1) I don't understand why you keep putting those PAUSE statements in there, now you have a PAUSE 15 in there....
2) There's no GOTO LOOP anywhere so the program will "fall thru" into the UpOrDn routine then hit the RETURN and bad things will happpen. (BTW, Loop is a reserved word in newer versions of PBP, best to get used to not using it.
3) Try resampling the pins and set oldState when you return from the subroutine, ie (changes in red):
Code:
Main:
    PortA.1 = 0
    newState = PortA & %00011100         ; this is for my hw : 11100 = 28
    PortA.1 = 1
    If newState <> 28 THEN
        If newState <> oldState THEN
              
            Select Case oldState
                Case 12
                If NewState = 20 THEN dir=up       
                If NewState = 24 THEn dir=dn
                
                Case 20
                If NewState = 24 THEN dir=up               
                If NewState = 12 THEN dir=dn
               
                Case 24
                If NewState = 12 THEN dir=up                       
                If NewState = 20 THEN dir=dn
            END SELECT
            GOSUB UpOrDn 
            
            PAUSE 15                    ; time for UpOrDn   
         
     ' Now resample the encoder and keep the state as oldState
          PortA.1 = 0
         oldState = PortA & %00011100         ; this is for my hw : 11100 = 28
         PortA.1 = 1     
       
    ENDIF          
  ENDIF
GOTO MAIN
;============
UpOrDn:
if dir=DN then
    PortB.4 = 0
    pauseus 3850
    PortB.4 = 1
    pauseus 3850
    PortB.4 = 0
    pauseus 660
    PortB.4 = 1
endif
if dir=UP then
    PortB.4 = 0
    pauseus 4450
    PortB.4 = 1
    pauseus 4400
    PortB.4 = 0
    pauseus 660
    PortB.4 = 1
endif    
RETURN
END