uKern based MX FPC high CPU utilization troubleshooting checklist

CPU utilization script:

set snmp rmon alarm 1 variable jnxOperatingCPU.7.1.0.0 <<< change this value based on the fpc slot number.

set snmp rmon alarm 1 sample-type absolute-value

set snmp rmon alarm 1 rising-threshold 70

set snmp rmon alarm 1 rising-event-index 1

set snmp rmon event 1 type log

set event-options policy test events snmpd_rmon_eventlog

set event-options policy test then execute-commands commands "show chassis fpc | no-more"

set event-options policy test then execute-commands commands "show system processes extensive | except \" 0.00% \" | no-more"

set event-options policy test then execute-commands commands "show policer | no-more"

set event-options policy test then execute-commands commands "show pfe statistics traffic fpc 0 | no-more"

set event-options policy test then execute-commands commands "show system statistics | no-more"

set event-options policy test then execute-commands commands "show route summary | no-more"

set event-options policy test then execute-commands commands "show system virtual-memory | no-more"

set event-options policy test then execute-commands commands "set task accounting on"

set event-options policy test then execute-commands commands "show task accounting detail | no-more" 

set event-options policy test then execute-commands commands "set task accounting off"

set event-options policy test then execute-commands commands "show task io | no-more"

set event-options policy test then execute-commands commands "show task statistics | no-more"

set event-options policy test then execute-commands commands "show task jobs | no-more"

set event-options policy test then execute-commands commands "show krt queue | no-more"

set event-options policy test then execute-commands commands "show krt state | no-more"

set event-options policy test then execute-commands commands "request pfe execute command \"show threads \" target fpc0"

set event-options policy test then execute-commands commands "request pfe execute command \"show threads verbose \" target fpc0"

set event-options policy test then execute-commands commands "request pfe execute command \"show threads cpu \" target fpc0"

set event-options policy test then execute-commands commands "request pfe execute command \"show sched \" target fpc0"

set event-options policy test then execute-commands commands "request pfe execute command \"show nhdb management ipc \"target fpc0"

set event-options policy test then execute-commands commands "request pfe execute command \"show pfe manager statistics \"target fpc0"

set event-options policy test then execute-commands commands "request pfe execute command \"show socket \" target fpc0"

set event-options policy test then execute-commands commands "request pfe execute command \"show route summary \" target fpc0"

set event-options policy test then execute-commands commands "request pfe execute command \"show nhdb summary \" target fpc0"

set event-options policy test then execute-commands commands "request pfe execute command \"show nhdb management all \" target fpc0"

set event-options policy test then execute-commands commands "request pfe execute command \"show nhdb mcast resolve \" target fpc0"

set event-options policy test then execute-commands commands "request pfe execute command \"show fpc interrupts \" target fpc0"

set event-options policy test then execute-commands commands "request pfe execute command \"show packet \" target fpc0"

set event-options policy test then execute-commands commands "request pfe execute command \"show packet statistics \" target fpc0"

set event-options policy test then execute-commands commands "request pfe execute command \"show ukern_trace memory-composition \" target fpc0"

set event-options policy test then execute-commands commands "request pfe execute command \"show ddos state \" target fpc0"

set event-options policy test then execute-commands commands "request pfe execute command \"show ddos asic global-rx-errors \" target fpc0"

set event-options policy test then execute-commands commands "request pfe execute command \"show ddos asic punt-proto-maps \" target fpc0"

set event-options policy test then execute-commands commands "request pfe execute command \"show ddos policer stats all \" target fpc0"

set event-options policy test then execute-commands commands "request pfe execute command \"show ddos policer stats all terse \" target fpc0"

set event-options policy test then execute-commands commands "request pfe execute command \"show syslog messages \" target fpc0"

set event-options policy test then execute-commands commands "request pfe execute command \"show nhdb management operations \" target fpc0"

set event-options policy test then execute-commands commands "request pfe execute command \"show nhdb management resolve \" target fpc0"

set event-options policy test then execute-commands commands "request pfe execute command \"show nhdb resolve throttle \" target fpc0"

set event-options policy test then execute-commands commands "request pfe execute command \"show route manager statistics \" target fpc0"

set event-options policy test then execute-commands commands "request pfe execute command \"show jnh packet \" target fpc0"

set event-options policy test then execute-commands commands "request pfe execute command \"show jnh host-path-stats \" target fpc0"

set event-options policy test then execute-commands commands "request pfe execute command \"show jnh if statistics \" target fpc0"

set event-options policy test then execute-commands commands "request pfe execute command \"show jnh 0 exceptions terse \" target fpc0"

set event-options policy test then execute-commands commands "request pfe execute command \"show jnh 0 exceptions hbc \" target fpc0"

set event-options policy test then execute-commands commands "request pfe execute command \"show jnh 1 exceptions terse \" target fpc0"

set event-options policy test then execute-commands commands "request pfe execute command \"show jnh 1 exceptions hbc \" target fpc0"

set event-options policy test then execute-commands commands "request pfe execute command \"show pfe manager session statistics \" target fpc0"

set event-options policy test then execute-commands commands "request pfe execute command \"show throttle \" target fpc0"

set event-options policy test then execute-commands commands "request pfe execute command \"show topo stat \" target fpc0"

set event-options policy test then execute-commands commands "request pfe execute command \"show sample summary \" target fpc0"

set event-options policy test then execute-commands commands "request pfe execute command \"show ddos policer sample stats \" target fpc0"

set event-options policy test then execute-commands commands "request pfe execute command \"show services inline-jflow summary \" target fpc0"

set event-options policy test then execute-commands commands "request pfe execute command \"show jnh 0 sample-inline statistics ipv4 \" target fpc0"

set event-options policy test then execute-commands commands "request pfe execute command \"show jnh 0 sample-inline statistics ipv6 \" target fpc0"

set event-options policy test then execute-commands commands "request pfe execute command \"show jnh 0 sample-inline statistics mpls \" target fpc0"

set event-options policy test then execute-commands commands "request pfe execute command \"show jnh 1 sample-inline statistics ipv4 \" target fpc0"

set event-options policy test then execute-commands commands "request pfe execute command \"show jnh 1 sample-inline statistics ipv6 \" target fpc0"

set event-options policy test then execute-commands commands "request pfe execute command \"show jnh 1 sample-inline statistics mpls \" target fpc0"

set event-options policy test then execute-commands commands "request pfe execute command \"show stack \" target fpc0"

set event-options policy test then execute-commands commands "request pfe execute command \"show fpc interrupts \" target fpc0"

set event-options policy test then execute-commands commands "request pfe execute command \"show ithrottle \" target fpc0"

set event-options policy test then execute-commands commands "request pfe execute command \"show pci interrupts \" target fpc0"

set event-options policy test then execute-commands commands "request pfe execute command \"show pfe manager statistics \" target fpc0"

set event-options policy test then execute-commands commands "request pfe execute command \"show pfe manager session statistics \" target fpc0"

set event-options policy test then execute-commands commands "request pfe execute command \"show pfe manager service-thread jflow statistics \" target fpc0"

set event-options policy test then execute-commands commands "request pfe execute command \"show pfe manager service-thread statistics \" target fpc0"

set event-options policy test then execute-commands commands "request pfe execute command \"show pfe manager service-queue \" target fpc0"

set event-options policy test then execute-commands output-filename FPCAggregateCPU-GET-HIGH.txt

set event-options policy test then execute-commands destination log-for-AggregateCPU-GET-HIGH

set event-options policy test then execute-commands output-format text

set event-options destinations log-for-AggregateCPU-GET-HIGH archive-sites /var/log/

• Additionally, we may investigate the RE to ukern communication for any issues.

root@mx2008:/var/home/# tcpdump -ni ixlv0

verbose output suppressed, use <detail> or <extensive> for full protocol decode

Address resolution is OFF.

Listening on ixlv0, capture size 96 bytes

labroot@mx2008> monitor traffic interface ixlv0 count 10000 size 4000 write-file /var/tmp/ixlv0.pcap << wait for sometime

Address resolution is ON. Use <no-resolve> to avoid any reverse lookup delay.

Address resolution timeout is 4s.

Listening on ixlv0, capture size 4000 bytes

• From the FPC where CPU usage stays high.

start shell pfe network fpc<slot number>

set parser security 10

clear sched

show sched

show fpc interrupt

show thread cpu

< wait for ~ 10 sec>

show sched

show fpc interrupt

show thread cpu

• If required , we may take rtsockmon -nrt at least for 1min. 

rtsockmon -nt > /var/tmp/rtsockmonfpc.txt

root@mx2008:/var/home/labroot # rtsockmon -nt > /var/tmp/rtsockmonfpc.txt