If I revert to the Old_Bits/New_Bits logic it now works:

Code:
DEFINE OSC 20            ' Set oscillator 20Mhz


' ***************************************************************
' Device Fuses
' ***************************************************************
#CONFIG
   __config _CONFIG1, _FOSC_HS & _WDTE_ON & _PWRTE_ON & _MCLRE_OFF & _CP_OFF & _CPD_OFF
   __config _CONFIG2, _PLLEN_OFF & _STVREN_ON & _BORV_LO & _LVP_OFF
#ENDCONFIG


' ***************************************************************
' Initialization
' ***************************************************************


DEFINE CCP3_REG PORTA
DEFINE CCP3_BIT 2


ANSELA.0 = 0             ' Digital only on roatary encoder B pin
ANSELA.1 = 0             ' Digital only on roatary encoder A pin
ANSELA.2 = 0             ' Digital only on CCP3 pin
TRISA    = %00000011     ' Make PortA pins 0-1 input for rotary encoder


INTCON   = %10001000     ' Global int enabled, IOCI enabled, IOCI flag bit 0 clr
IOCAP.0  = 1             ' Enable positive (rising edge) change
IOCAP.1  = 1             ' Enable positive (rising edge) change
IOCAN.0  = 1             ' Enable negative (falling edge) change
IOCAN.1  = 1             ' Enable negative (falling edge) change
IOCAF.0  = 0             ' Clear interupt-on-change flag
IOCAF.1  = 0             ' Clear interupt-on-change flag


Old_Bits       VAR BYTE
New_Bits       VAR BYTE
RotEncDir      VAR BIT   ' 1=CW, 0=CCW
                         ' Rot Enc pin A connected to PortA.1;
                         ' Rot Enc pin B connected to PortA.0
Old_RPM        VAR BYTE
I              VAR BYTE


' ***************************************************************
' Includes
' ***************************************************************


INCLUDE "EE_Vars.PBP"       ' Requires MPASM assembler
                            ' Go to "View > Compile and Program Options..."
                            ' On "Assembler" tab, check "Use MPASM"
                            ' (no need to change any of the default settings)
                            ' --> copy file to PBP main folder (i.e. c:\pbp)


MotorRPM VAR BYTE        :  @  EE_var        _MotorRPM, BYTE, 100
;PortNacelleDir VAR BYTE  :  @  EE_var  _PortNacelleDir, BYTE, 0   ' 0 = CCW (inwards)
                                                                  ' 1 = CW (outwards)


INCLUDE "DT_INTS-14.bas"     ' Base Interrupt System
INCLUDE "ReEnterPBP.bas"     ' Include if using PBP interrupts
                             ' --> copy both files to PBP main folder 
                             ' (i.e. c:\pbp)
ASM
INT_LIST  macro    ; IntSource,         Label,  Type, ResetFlag?
        INT_Handler    IOC_INT,  _Rot_Encoder,   PBP,  yes
    endm
    INT_CREATE     ; Creates the interrupt processor
ENDASM


@ INT_ENABLE   IOC_INT     ; Interrupt-on-Change interrupt


' Set default values
Old_RPM = MotorRPM
Old_Bits = PORTA & (%00000011)


' Spin up motor to saved value of _MotorRPM
IF MotorRPM > 1 THEN
    FOR I = 0 to MotorRPM
        pause 30
        HPWM 3, I, 20000
    NEXT I
EndIf


Main:
    IF MotorRPM <> Old_RPM Then
        Old_RPM = MotorRPM
        GOSUB motorhpwm
@ EE_write_var _MotorRPM        ; save the new number to EEPROM
    EndIF   
    
    GOTO Main


motorhpwm:
    HPWM 3, MotorRPM, 20000 ; Tried 245 Hz but it made the motor too loud.
                            ; Supposedly, anything above 20kHz is above 
                            ; human hearing


    RETURN
end


' ***************************************************************
' [IOC - interrupt handler]
' ***************************************************************
Rot_Encoder:
    New_Bits = PORTA & (%00000011)
    IF (New_Bits & %00000011) = (Old_Bits & %00000011) Then DoneRotEnc
    RotEncDir = New_Bits.1 ^ Old_Bits.0
    IF RotEncDir = 1 Then
        ; CW rotation - increase speed but only to a max of 255
        IF MotorRPM < 255 then MotorRPM = MotorRPM + 1
    Else
        ' CCW rotation - decrease speed to a min of 0
        IF MotorRPM > 0 Then MotorRPM = MotorRPM - 1
    EndIF


DoneRotEnc:
    Old_Bits = New_Bits
    
    IOCAF.0 = 0  ' Clear interrupt flags
    IOCAF.1 = 0


@ INT_RETURN