Henrik assuming i have implemented your ideascorrectly it works not at all well for a ky040 re in real life, i get about 400 counts per indent and rarely does it get direction correct.
my old code previously posted works near perfectly in the same rig either as isr or polled even a couple of .1uf across the pins does not help.


Code:
#CONFIG
             __config        _CONFIG1,    _FOSC_INTOSC & _CP_OFF & _WDTE_ON  &  _PWRTE_ON  &  _MCLRE_ON  & _CLKOUTEN_OFF
              __config      _CONFIG2, _PLLEN_ON & _LVP_OFF
#ENDCONFIG


DEFINE OSC 32
    OSCCON=$70
    ANSELA=0
    OPTION_REG.6=0
    ENCODER    VAR BYTE   ;THE STATE OF THE ENCODER
    DIR   VAR BYTE    ;POSN
    Old_Enc  VAR BYTE    ;LAST MOVE 
    TEST   VAR BYTE 
    Position   VAR WORD 
    TRISA = %11111110
    lata.0=1 ;DEBUG
    Ch_A    VAR PORTA.4
    Ch_B    VAR PORTA.5
    
    
    DEFINE DEBUG_REG PORTA
    DEFINE DEBUG_BIT 0        
    DEFINE DEBUG_BAUD 38400
    DEFINE DEBUG_MODE 0     
    pause 2000
    Debug "Start",13 ,10
    Position =0 
    Old_Enc=0
    
mainloop:
'     Encoder.0= Ch_A    ;WON'T WORK AT ALL MOST TIMES
'     Encoder.1= Ch_B 
    Encoder = (PORTA&48)>>4
    Test = Encoder ^ Old_Enc
    Dir = 0
    If Test = 1 THEN
        If Old_Enc = 0 THEN Dir = 1
        IF Old_Enc = 1 THEN DIR = -1
        IF Old_Enc = 2 THEN Dir = -1
        If Old_Enc = 3 Then Dir = 1
    ENDIF
    IF Test = 2 THEN
        IF Old_Enc = 0 THEN DIR = -1
        IF Old_Enc = 1 THEN Dir = 1
        IF Old_Enc = 2 THEN Dir = 1
        IF Old_Enc = 3 THEN Dir = -1
    ENDIF
    Old_Enc = Encoder
    Position = Position + Dir
    IF DIR THEN 
        debug 13,10, "C",SDEC Position
    ENDIF
goto mainloop


my take [no pic18 worky or shitty old chips with no wreg access]

Code:
'****************************************************************'*  Name    : RE16.BAS                                          *
'*  Author  : RICHARD                                           *
'*  Notice  :                                                   *
'*          : All Rights Reserved                               *
'*  Date    : 5/06/2020                                         *
'*  Version : 1.0                                               *
'*  Notes   :    ONLY PIC16'S AND 12'S WITH WREG                *
'*          :   IN THIS INCARNTATION NO SHITTY OLD CHIPS        *
'****************************************************************


#CONFIG
             __config        _CONFIG1,    _FOSC_INTOSC & _CP_OFF & _WDTE_ON  &  _PWRTE_ON  &  _MCLRE_ON  & _CLKOUTEN_OFF
              __config      _CONFIG2, _PLLEN_ON & _LVP_OFF
#ENDCONFIG


DEFINE OSC 32
    OSCCON=$70
    ANSELA=0
    OPTION_REG.6=0
    ev    VAR BYTE bank0   ;THE STATE OF THE ENCODER
    cnt   VAR WORD bank0   ;POSN
    TMP   VAR BYTE bank0   ;LAST MOVE IS 1, 0, -1
    TRISA = %11111110
    lata.0=1 ;DEBUG
    
    DEFINE DEBUG_REG PORTA
    DEFINE DEBUG_BIT 0        
    DEFINE DEBUG_BAUD 38400
    DEFINE DEBUG_MODE 0     
    pause 2000
    Debug "Start",13 ,10
    CNT=0 


mainloop:
ASM
    bcf STATUS,C      
    RLF _ev ,f        ;shift last reading into position
    bcf STATUS,C
    RLF _ev ,f
    MOVF PORTA,W     ;read enc pins 4,5
    ANDLW 48         ;mask off others
    SWAPF   WREG ,w  ;shift NEW reading into position
    IORWF _ev,F     ;combine this read with last read
    MOVF _ev,W     ;save ev
    ANDLW 15
    L?CALL enc_lut       ; decide to inc , dec or ignore 
    MOVWF _TMP         ;save decision
    BTFSC  _TMP,7
    GOTO myDEC16
    BTFSC  _TMP,0
    GOTO myINC16
enc_exit                    ;  exit
    GOTO OUT
myINC16
    INCF _cnt ,f
    BTFSC STATUS,Z
    INCF  _cnt +1 ,f 
    GOTO enc_exit
myDEC16
    MOVF  _cnt ,w
    BTFSC STATUS,Z
    DECF   _cnt +1, f   
    DECF   _cnt , f       
    GOTO enc_exit
OUT    
    ENDASM
    IF TMP THEN 
        debug 13,10,SDEC CNT
    ENDIF
goto mainloop


enc:
ASM
enc_lut                     ; the decision matrix
    addwf   PCL, F          
    retlw   0
    retlw   255
    retlw   1
    retlw   0
    retlw   1
    retlw   0
    retlw   0
    retlw   255
    retlw   255
    retlw   0
    retlw   0
    retlw   1
    retlw   0
    retlw   1
    retlw   255
    retlw   0  
ENDASM