AT91SAM7SE256 hangs somewhere, doing USB

I suspect that the UDP interrupt hangs somewhere when I cycle the USB connection.

This is one construct that I use to set/clear bits in the CSR registers:

#define CLEAR_CSR(endpoint, flags) \
    { \
        volatile unsigned int reg; \
        reg = AT91C_BASE_UDP->UDP_CSR[endpoint]; \
        reg |= (AT91C_UDP_RX_DATA_BK0 | AT91C_UDP_RX_DATA_BK1); \
        reg &= ~(flags); \
        AT91C_BASE_UDP->UDP_CSR[endpoint] = reg; \
        while ( (AT91C_BASE_UDP->UDP_CSR[endpoint] & (flags)) == (flags) ) \
        {   \
            AT91C_BASE_UDP->UDP_CSR[endpoint] = reg; \
        }   \
    }

#define SET_CSR(endpoint, flags) \
    { \
        volatile unsigned int reg; \
        reg = AT91C_BASE_UDP->UDP_CSR[endpoint] ; \
        reg |= (flags); \
        reg |= (AT91C_UDP_RX_DATA_BK0 | AT91C_UDP_RX_DATA_BK1); \
        AT91C_BASE_UDP->UDP_CSR[endpoint] = reg; \
        while ( (AT91C_BASE_UDP->UDP_CSR[endpoint] & (flags)) != (flags) ) \
        {   \
            AT91C_BASE_UDP->UDP_CSR[endpoint] = reg; \
        }   \
    }

Am I doing this right? I am afraid that I might get stuck here when pulling/reinserting the USB cableā€¦