A few hints that might help based on my own use of API mode. Here is the setup process for API mode 2 (transparent):
To enter API mode:
Wait 1.2 secs
Send +++
Wait 1.2 secs
Send ATAP2,CN
Wait 0.2 secs
Here is the assembly code I use to build the API frame:
; setup an xbee api frame
; exit: zl/zh=buffer pointer
XBeeApiBeg: ENTRY
ldi zl,low(XBeeOutBuf+2)
ldi zh,high(XBeeOutBuf+2)
clr r16
st z+,r16 ; length=0
dec r16
st z+,r16 ; checksum=$ff
ret
; setup for broadcast packet
XBeeApiBrd: ENTRY
ldi r16,$01
rcall XBeeApiAdd ; tx16
ldi r16,$00
rcall XBeeApiAdd ; no ack
ser r16
rcall XBeeApiAdd
rcall XBeeApiAdd ; dest=broadcast
ldi r16,$04
rcall XBeeApiAdd ; broadcast mode
ret
; append a byte to the frame
; entry: r16=byte
XBeeApiAdd: ENTRY
push r17
lds r17,XBeeOutBuf+2
inc r17 ; count the byte
sts XBeeOutBuf+2,r17
lds r17,XBeeOutBuf+3
sub r17,r16 ; add byte to checksum
sts XBeeOutBuf+3,r17
mov r17,r16
cpi r17,$7e
breq xbeeapiadd2
cpi r17,$7d
breq xbeeapiadd2
cpi r17,$13
breq xbeeapiadd2
cpi r17,$11
brne xbeeapiadd3
xbeeapiadd2:ldi r17,$7d ; append escape character
st z+,r17
ldi r17,$20
eor r17,r16 ; escape the data byte
xbeeapiadd3:st z+,r17
pop r17
ret
; complete the frame
; exit: zl/zh=buffer pointer, r16=length
XBeeApiEnd: ENTRY
lds r16,XBeeOutBuf+3
rcall XBeeApiAdd ; append the checksum
push zl
push zh ; save encoded buffer end
ldi zl,low(XBeeOutBuf+4)
ldi zh,high(XBeeOutBuf+4)
lds r16,XBeeOutBuf+2 ; get raw count
dec r16 ; exclude checksum
cpi r16,$7e
breq xbeeapiend2
cpi r16,$7d
breq xbeeapiend2
cpi r16,$13
breq xbeeapiend2
cpi r16,$11
brne xbeeapiend3
xbeeapiend2:ldi r17,$20
eor r16,r17
st -z,r16 ; save escaped low-length
ldi r16,$7d
xbeeapiend3:st -z,r16 ; save low-length
clr r16
st -z,r16 ; save high-length
ldi r16,$7e
st -z,r16 ; save frame marker
pop r17
pop r16 ; get back end pointer
sub r16,zl
sbc r17,zh ; r16=length, zl/zh=buffer pointer
ret
To build an API frame, call the begin function, the broadcast header function (you can replace with a unicast destination) and then the data add function. After adding all data, call the end function which will return with zl/zh pointing to the completed packet (length in r16). Send it as-is to the XBee and you are good to go.
rcall XBeeApiBeg ; build a frame
rcall XBeeApiBrd ; broadcast data
ldi r16,'D'
rcall XBeeApiAdd ; data
lds r16,SetupNode
rcall XBeeApiAdd ; node-id
lds r16,SetupGroup
rcall XBeeApiAdd ; group-id
ldi r16,'F'
rcall XBeeApiAdd ; type=feeder
lds r16,FeederStop
rcall XBeeApiAdd ; data=duration
rcall XBeeApiEnd