(conanenv) prateek@prateek:~/Workspace/UCX/build$ ./examples/client.bin [1749297901.730722] [prateek:19822:0] debug.c:1157 UCX DEBUG using signal stack 0x74059caf4000 size 141824 [1749297901.731429] [prateek:19822:0] cpu.c:338 UCX DEBUG measured tsc frequency 3398.882 MHz after 0.36 ms [1749297901.731450] [prateek:19822:0] init.c:120 UCX DEBUG /lib/libucs.so.0 loaded at 0x74059cf28000 [1749297901.731476] [prateek:19822:0] init.c:122 UCX DEBUG cmd line: ./examples/client.bin [1749297901.731488] [prateek:19822:0] module.c:72 UCX DEBUG ucs library path: /lib/libucs.so.0 [1749297901.731495] [prateek:19822:0] module.c:280 UCX DEBUG loading modules for ucs [1749297901.731654] [prateek:19822:0] time.c:22 UCX DEBUG arch clock frequency: 3398882191.78 Hz UCX_NET_DEVICES=all UCX_SHM_DEVICES=all UCX_ACC_DEVICES=all UCX_SELF_DEVICES=all UCX_TLS=all UCX_PROTOS=all UCX_ALLOC_PRIO=md:sysv,md:posix,thp,md:*,mmap,heap UCX_RNDV_FRAG_SIZE=host:512K,cuda:4M UCX_RNDV_FRAG_ALLOC_COUNT=host:128,cuda:128 UCX_SOCKADDR_TLS_PRIORITY=rdmacm,tcp,sockcm UCX_WARN_INVALID_CONFIG=y UCX_RX_MPOOL_SIZES=64,1K UCX_RCACHE_ENABLE=try UCX_RCACHE_MEM_PRIO=500 UCX_RCACHE_OVERHEAD=auto UCX_RCACHE_MAX_REGIONS=inf UCX_RCACHE_MAX_SIZE=inf UCX_RCACHE_MAX_UNRELEASED=inf UCX_RCACHE_PURGE_ON_FORK=y UCX_SELECT_DISTANCE_MD=cuda_cpy UCX_MEMTYPE_REG_WHOLE_ALLOC_TYPES=cuda UCX_RNDV_MEMTYPE_DIRECT_SIZE=inf UCX_BCOPY_THRESH=auto UCX_RNDV_THRESH=intra:auto,inter:auto UCX_RNDV_SEND_NBR_THRESH=256K UCX_RNDV_THRESH_FALLBACK=inf UCX_RNDV_PERF_DIFF=1.000 UCX_MULTI_LANE_MAX_RATIO=4.000 UCX_MULTI_PATH_RATIO=auto UCX_MAX_EAGER_RAILS=1 UCX_MAX_RNDV_RAILS=2 UCX_MAX_RMA_RAILS=1 UCX_MIN_RNDV_CHUNK_SIZE=16K UCX_RMA_ZCOPY_MAX_SEG_SIZE=auto UCX_RNDV_SCHEME=auto UCX_RKEY_PTR_SEG_SIZE=512K UCX_ZCOPY_THRESH=auto UCX_BCOPY_BW=auto UCX_ATOMIC_MODE=guess UCX_ADDRESS_DEBUG_INFO=n UCX_MAX_WORKER_ADDRESS_NAME=32 UCX_USE_MT_MUTEX=n UCX_ADAPTIVE_PROGRESS=y UCX_SEG_SIZE=8K UCX_TM_THRESH=1K UCX_TM_MAX_BB_SIZE=1K UCX_TM_FORCE_THRESH=8K UCX_TM_SW_RNDV=n UCX_NUM_EPS=auto UCX_NUM_PPN=auto UCX_RNDV_FRAG_MEM_TYPES=host,cuda UCX_RNDV_PIPELINE_SEND_THRESH=inf UCX_RNDV_PIPELINE_SHM_ENABLE=y UCX_RNDV_PIPELINE_ERROR_HANDLING=n UCX_FLUSH_WORKER_EPS=y UCX_FENCE_MODE=auto UCX_UNIFIED_MODE=n UCX_CM_USE_ALL_DEVICES=y UCX_LISTENER_BACKLOG=auto UCX_PROTO_ENABLE=y UCX_PROTO_REQUEST_RESET=n UCX_KEEPALIVE_INTERVAL=20000000.00us UCX_KEEPALIVE_NUM_EPS=128 UCX_DYNAMIC_TL_SWITCH_INTERVAL=inf UCX_DYNAMIC_TL_PROGRESS_FACTOR=1154512.17us UCX_RESOLVE_REMOTE_EP_ID=off UCX_PROTO_INDIRECT_ID=auto UCX_RNDV_PUT_FORCE_FLUSH=n UCX_SA_DATA_VERSION=v2 UCX_RKEY_MPOOL_MAX_MD=2 UCX_ADDRESS_VERSION=v1 UCX_PROTO_INFO=n UCX_RNDV_ALIGN_THRESH=64K UCX_PROTO_INFO_DIR= UCX_REG_NONBLOCK_MEM_TYPES= UCX_PREFER_OFFLOAD=y UCX_PROTO_OVERHEAD=single:0.01us,multi:0.01us,rndv_offload:0.04us,rndv_rtr:0.04us,rndv_rts:0.28us,sw:0.04us,rkey_ptr:0.00us UCX_GVA_ENABLE=off UCX_GVA_MLOCK=y UCX_GVA_PREFETCH=y [1749297901.731956] [prateek:19822:0] ucp_context.c:2257 UCX INFO Version 1.18.1 (loaded from /lib/libucp.so.0) [1749297901.731994] [prateek:19822:0] ucp_context.c:2015 UCX DEBUG estimated number of endpoints is 1 [1749297901.731999] [prateek:19822:0] ucp_context.c:2022 UCX DEBUG estimated number of endpoints per node is 1 [1749297901.732003] [prateek:19822:0] ucp_context.c:2033 UCX DEBUG estimated bcopy bandwidth is 7340032000.000000 [1749297901.732013] [prateek:19822:0] ucp_context.c:2092 UCX DEBUG allocation method[0] is md 'sysv' [1749297901.732016] [prateek:19822:0] ucp_context.c:2092 UCX DEBUG allocation method[1] is md 'posix' [1749297901.732024] [prateek:19822:0] ucp_context.c:2104 UCX DEBUG allocation method[2] is 'thp' [1749297901.732028] [prateek:19822:0] ucp_context.c:2092 UCX DEBUG allocation method[3] is md '*' [1749297901.732032] [prateek:19822:0] ucp_context.c:2104 UCX DEBUG allocation method[4] is 'mmap' [1749297901.732036] [prateek:19822:0] ucp_context.c:2104 UCX DEBUG allocation method[5] is 'heap' [1749297901.732049] [prateek:19822:0] module.c:280 UCX DEBUG loading modules for uct [1749297901.735281] [prateek:19822:0] module.c:280 UCX DEBUG loading modules for uct_cuda [1749297901.735735] [prateek:19822:0] module.c:280 UCX DEBUG loading modules for uct_ib [1749297901.736176] [prateek:19822:0] module.c:258 UCX DEBUG dlopen('/lib/ucx/libuct_rdmacm.so.0', mode=0x1) failed: librdmacm.so.1: cannot open shared object file: No such file or directory [1749297901.736276] [prateek:19822:0] module.c:258 UCX DEBUG dlopen('/usr/lib/ucx/libuct_rdmacm.so.0', mode=0x1) failed: librdmacm.so.1: cannot open shared object file: No such file or directory [1749297901.736369] [prateek:19822:0] module.c:258 UCX DEBUG dlopen('/lib/ucx/libuct_xpmem.so.0', mode=0x1) failed: /lib/ucx/libuct_xpmem.so.0: cannot open shared object file: No such file or directory [1749297901.736381] [prateek:19822:0] module.c:258 UCX DEBUG dlopen('/usr/lib/ucx/libuct_xpmem.so.0', mode=0x1) failed: /usr/lib/ucx/libuct_xpmem.so.0: cannot open shared object file: No such file or directory [1749297901.736409] [prateek:19822:0] cuda_md.c:76 UCX DIAG cuDeviceGetCount(&num_gpus) failed: initialization error [1749297901.736417] [prateek:19822:0] cuda_md.c:76 UCX DIAG cuDeviceGetCount(&num_gpus) failed: initialization error [1749297901.736473] [prateek:19822:0] ib_md.c:827 UCX DEBUG no devices are found [1749297901.736499] [prateek:19822:0] ib_md.c:827 UCX DEBUG no devices are found [1749297901.737025] [prateek:19822:0] topo.c:898 UCX DEBUG /sys/class/net/enp34s0: PF sysfs path is '/sys/devices/pci0000:00/0000:00:01.3/0000:03:00.2/0000:20:01.0/0000:22:00.0' [1749297901.737081] [prateek:19822:0] numa.c:153 UCX DEBUG failed to discover numa node for device: /sys/devices/pci0000:00/0000:00:01.3/0000:03:00.2/0000:20:01.0/0000:22:00.0, status Success, parsed_node -1 [1749297901.737087] [prateek:19822:0] topo.c:298 UCX DEBUG added sys_dev 0 for bus id 22:00.0 [1749297901.737093] [prateek:19822:0] topo.c:552 UCX DEBUG enp34s0: bdf_name 0000:22:00.0 sys_dev 0 [1749297901.737265] [prateek:19822:0] topo.c:895 UCX DEBUG /sys/class/net/lo: sysfs path undetected [1749297901.737271] [prateek:19822:0] topo.c:556 UCX DEBUG lo: system device unknown [1749297901.737334] [prateek:19822:0] cuda_md.c:76 UCX DIAG cuDeviceGetCount(&num_gpus) failed: initialization error [1749297901.737340] [prateek:19822:0] cuda_md.c:76 UCX DIAG cuDeviceGetCount(&num_gpus) failed: initialization error [1749297901.737362] [prateek:19822:0] ib_md.c:827 UCX DEBUG no devices are found [1749297901.737381] [prateek:19822:0] ib_md.c:827 UCX DEBUG no devices are found [1749297901.737410] [prateek:19822:0] ucp_context.c:1712 UCX DEBUG register host memory on: self, tcp, cma [1749297901.737415] [prateek:19822:0] ucp_context.c:1700 UCX DEBUG no memory domain supports registering cuda memory [1749297901.737418] [prateek:19822:0] ucp_context.c:1700 UCX DEBUG no memory domain supports registering cuda-managed memory [1749297901.737422] [prateek:19822:0] ucp_context.c:1700 UCX DEBUG no memory domain supports registering rocm memory [1749297901.737427] [prateek:19822:0] ucp_context.c:1700 UCX DEBUG no memory domain supports registering rocm-managed memory [1749297901.737432] [prateek:19822:0] ucp_context.c:1700 UCX DEBUG no memory domain supports registering rdma memory [1749297901.737437] [prateek:19822:0] ucp_context.c:1700 UCX DEBUG no memory domain supports registering ze-host memory [1749297901.737440] [prateek:19822:0] ucp_context.c:1700 UCX DEBUG no memory domain supports registering ze-device memory [1749297901.737446] [prateek:19822:0] ucp_context.c:1700 UCX DEBUG no memory domain supports registering ze-managed memory [1749297901.737465] [prateek:19822:0] mpool.c:136 UCX DEBUG mpool rcache_mp: align 8, maxelems 4294967295, elemsize 144 [1749297901.737489] [prateek:19822:0] async.c:247 UCX DEBUG added async handler 0x63dfe09ceb50 [id=3 ref 1] ???() to hash [1749297901.737600] [prateek:19822:0] async.c:521 UCX DEBUG listening to async event fd 3 events 0x1 mode thread_spinlock [1749297901.737763] [prateek:19822:0] module.c:280 UCX DEBUG loading modules for ucm [1749297901.803095] [prateek:19822:0] ucp_context.c:2329 UCX DEBUG created ucp context ucp_server 0x63dfe09b0ae0 [5 mds 6 tls] features 0xe3 tl bitmap 0x3f 0x0 [1749297901.803209] [prateek:19822:0] mpool.c:136 UCX DEBUG mpool self_msg_desc: align 64, maxelems 4294967295, elemsize 8200 [1749297901.803218] [prateek:19822:0] self.c:245 UCX DEBUG created self iface id 0xe6591fb9d07f2edd send_size 8192 [1749297901.803232] [prateek:19822:0] ucp_worker.c:1434 UCX DEBUG created interface[0]=0x63dfe09b0410 using self/memory on worker 0x63dfe09d38a0 [1749297901.803256] [prateek:19822:0] mpool.c:136 UCX DEBUG mpool uct_tcp_iface_tx_buf_mp: align 64, maxelems 4294967295, elemsize 8205 [1749297901.803261] [prateek:19822:0] mpool.c:136 UCX DEBUG mpool uct_tcp_iface_rx_buf_mp: align 64, maxelems 4294967295, elemsize 131090 [1749297901.803477] [prateek:19822:0] async.c:247 UCX DEBUG added async handler 0x63dfe09cc150 [id=9 ref 1] ???() to hash [1749297901.803492] [prateek:19822:0] async.c:521 UCX DEBUG listening to async event fd 9 events 0x5 mode thread_spinlock [1749297901.803505] [prateek:19822:0] tcp_iface.c:596 UCX DEBUG tcp_iface 0x63dfe09aeba0: listening for connections (fd=9) on 192.168.0.103:47395 netif enp34s0 [1749297901.803626] [prateek:19822:0] topo.c:898 UCX DEBUG /sys/class/net/enp34s0: PF sysfs path is '/sys/devices/pci0000:00/0000:00:01.3/0000:03:00.2/0000:20:01.0/0000:22:00.0' [1749297901.803895] [prateek:19822:0] ucp_worker.c:1434 UCX DEBUG created interface[1]=0x63dfe09aeba0 using tcp/enp34s0 on worker 0x63dfe09d38a0 [1749297901.803913] [prateek:19822:0] mpool.c:136 UCX DEBUG mpool uct_tcp_iface_tx_buf_mp: align 64, maxelems 4294967295, elemsize 8205 [1749297901.803917] [prateek:19822:0] mpool.c:136 UCX DEBUG mpool uct_tcp_iface_rx_buf_mp: align 64, maxelems 4294967295, elemsize 131090 [1749297901.804039] [prateek:19822:0] async.c:247 UCX DEBUG added async handler 0x63dfe09cc560 [id=11 ref 1] ???() to hash [1749297901.804048] [prateek:19822:0] async.c:521 UCX DEBUG listening to async event fd 11 events 0x5 mode thread_spinlock [1749297901.804052] [prateek:19822:0] tcp_iface.c:596 UCX DEBUG tcp_iface 0x63dfe09f6520: listening for connections (fd=11) on 127.0.0.1:54857 netif lo [1749297901.804069] [prateek:19822:0] sock.c:90 UCX DEBUG ioctl(req=35142, ifr_name=lo) failed: Operation not supported [1749297901.804077] [prateek:19822:0] tcp_net.c:61 UCX DEBUG speed of lo is UNKNOWN, assuming 100 Mbps [1749297901.804139] [prateek:19822:0] topo.c:895 UCX DEBUG /sys/class/net/lo: sysfs path undetected [1749297901.804144] [prateek:19822:0] topo.c:848 UCX DEBUG lo: pci bandwidth undetected, using maximal value [1749297901.804172] [prateek:19822:0] ucp_worker.c:1434 UCX DEBUG created interface[2]=0x63dfe09f6520 using tcp/lo on worker 0x63dfe09d38a0 [1749297901.804230] [prateek:19822:0] mm_sysv.c:97 UCX DEBUG mm failed to allocate 33023 bytes with hugetlb [1749297901.804284] [prateek:19822:0] mpool.c:136 UCX DEBUG mpool mm_recv_desc: align 64, maxelems 4294967295, elemsize 8368 [1749297901.804297] [prateek:19822:0] mm_sysv.c:97 UCX DEBUG mm failed to allocate 4292720 bytes with hugetlb [1749297901.804319] [prateek:19822:0] mpool.c:281 UCX DEBUG mpool mm_recv_desc: allocated chunk 0x740598de6018 of 4296680 bytes with 512 elements [1749297901.806114] [prateek:19822:0] mm_iface.c:730 UCX DEBUG created mm iface 0x63dfe09f74b0 FIFO id 0x20022 va 0x74059ca1f000 size 36864 (128 x 256 elems) [1749297901.806141] [prateek:19822:0] ucp_worker.c:1434 UCX DEBUG created interface[3]=0x63dfe09f74b0 using sysv/memory on worker 0x63dfe09d38a0 [1749297901.806270] [prateek:19822:0] mm_posix.c:572 UCX DEBUG allocated posix shared memory at 0x74059ca16000 length 36864 [1749297901.806289] [prateek:19822:0] mpool.c:136 UCX DEBUG mpool mm_recv_desc: align 64, maxelems 4294967295, elemsize 8368 [1749297901.808719] [prateek:19822:0] mm_posix.c:338 UCX DEBUG shared memory mmap(addr=(nil), length=6291456, flags= HUGETLB, fd=15) failed: Invalid argument [1749297901.808746] [prateek:19822:0] mm_posix.c:572 UCX DEBUG allocated posix shared memory at 0x7405989cd000 length 4296704 [1749297901.808759] [prateek:19822:0] mpool.c:281 UCX DEBUG mpool mm_recv_desc: allocated chunk 0x7405989cd018 of 4296680 bytes with 512 elements [1749297901.809417] [prateek:19822:0] mm_iface.c:730 UCX DEBUG created mm iface 0x63dfe09f8160 FIFO id 0xc000000340004d6e va 0x74059ca16000 size 36864 (128 x 256 elems) [1749297901.809430] [prateek:19822:0] ucp_worker.c:1434 UCX DEBUG created interface[4]=0x63dfe09f8160 using posix/memory on worker 0x63dfe09d38a0 [1749297901.809465] [prateek:19822:0] mpool.c:136 UCX DEBUG mpool uct_scopy_iface_tx_mp: align 64, maxelems 4294967295, elemsize 736 [1749297901.809476] [prateek:19822:0] ucp_worker.c:1434 UCX DEBUG created interface[5]=0x63dfe09f8d60 using cma/memory on worker 0x63dfe09d38a0 [1749297901.809484] [prateek:19822:0] ucp_worker.c:1151 UCX DEBUG selected scalable tl bitmap: 0x3f 0x0 (6 tls) [1749297901.809515] [prateek:19822:0] async.c:247 UCX DEBUG added async handler 0x63dfe09b01c0 [id=8 ref 1] ???() to hash [1749297901.809530] [prateek:19822:0] async.c:521 UCX DEBUG listening to async event fd 8 events 0x0 mode thread_spinlock [1749297901.809666] [prateek:19822:0] topo.c:898 UCX DEBUG /sys/class/net/enp34s0: PF sysfs path is '/sys/devices/pci0000:00/0000:00:01.3/0000:03:00.2/0000:20:01.0/0000:22:00.0' [1749297901.809817] [prateek:19822:0] topo.c:898 UCX DEBUG /sys/class/net/enp34s0: PF sysfs path is '/sys/devices/pci0000:00/0000:00:01.3/0000:03:00.2/0000:20:01.0/0000:22:00.0' [1749297901.809950] [prateek:19822:0] topo.c:898 UCX DEBUG /sys/class/net/enp34s0: PF sysfs path is '/sys/devices/pci0000:00/0000:00:01.3/0000:03:00.2/0000:20:01.0/0000:22:00.0' [1749297901.810076] [prateek:19822:0] topo.c:898 UCX DEBUG /sys/class/net/enp34s0: PF sysfs path is '/sys/devices/pci0000:00/0000:00:01.3/0000:03:00.2/0000:20:01.0/0000:22:00.0' [1749297901.810201] [prateek:19822:0] topo.c:898 UCX DEBUG /sys/class/net/enp34s0: PF sysfs path is '/sys/devices/pci0000:00/0000:00:01.3/0000:03:00.2/0000:20:01.0/0000:22:00.0' [1749297901.810325] [prateek:19822:0] topo.c:898 UCX DEBUG /sys/class/net/enp34s0: PF sysfs path is '/sys/devices/pci0000:00/0000:00:01.3/0000:03:00.2/0000:20:01.0/0000:22:00.0' [1749297901.810449] [prateek:19822:0] topo.c:898 UCX DEBUG /sys/class/net/enp34s0: PF sysfs path is '/sys/devices/pci0000:00/0000:00:01.3/0000:03:00.2/0000:20:01.0/0000:22:00.0' [1749297901.810573] [prateek:19822:0] topo.c:898 UCX DEBUG /sys/class/net/enp34s0: PF sysfs path is '/sys/devices/pci0000:00/0000:00:01.3/0000:03:00.2/0000:20:01.0/0000:22:00.0' [1749297901.810697] [prateek:19822:0] topo.c:898 UCX DEBUG /sys/class/net/enp34s0: PF sysfs path is '/sys/devices/pci0000:00/0000:00:01.3/0000:03:00.2/0000:20:01.0/0000:22:00.0' [1749297901.810822] [prateek:19822:0] topo.c:898 UCX DEBUG /sys/class/net/enp34s0: PF sysfs path is '/sys/devices/pci0000:00/0000:00:01.3/0000:03:00.2/0000:20:01.0/0000:22:00.0' [1749297901.810954] [prateek:19822:0] topo.c:898 UCX DEBUG /sys/class/net/enp34s0: PF sysfs path is '/sys/devices/pci0000:00/0000:00:01.3/0000:03:00.2/0000:20:01.0/0000:22:00.0' [1749297901.811079] [prateek:19822:0] topo.c:898 UCX DEBUG /sys/class/net/enp34s0: PF sysfs path is '/sys/devices/pci0000:00/0000:00:01.3/0000:03:00.2/0000:20:01.0/0000:22:00.0' [1749297901.811204] [prateek:19822:0] topo.c:898 UCX DEBUG /sys/class/net/enp34s0: PF sysfs path is '/sys/devices/pci0000:00/0000:00:01.3/0000:03:00.2/0000:20:01.0/0000:22:00.0' [1749297901.811328] [prateek:19822:0] topo.c:898 UCX DEBUG /sys/class/net/enp34s0: PF sysfs path is '/sys/devices/pci0000:00/0000:00:01.3/0000:03:00.2/0000:20:01.0/0000:22:00.0' [1749297901.811452] [prateek:19822:0] topo.c:898 UCX DEBUG /sys/class/net/enp34s0: PF sysfs path is '/sys/devices/pci0000:00/0000:00:01.3/0000:03:00.2/0000:20:01.0/0000:22:00.0' [1749297901.811578] [prateek:19822:0] topo.c:898 UCX DEBUG /sys/class/net/enp34s0: PF sysfs path is '/sys/devices/pci0000:00/0000:00:01.3/0000:03:00.2/0000:20:01.0/0000:22:00.0' [1749297901.811702] [prateek:19822:0] topo.c:898 UCX DEBUG /sys/class/net/enp34s0: PF sysfs path is '/sys/devices/pci0000:00/0000:00:01.3/0000:03:00.2/0000:20:01.0/0000:22:00.0' [1749297901.811826] [prateek:19822:0] topo.c:898 UCX DEBUG /sys/class/net/enp34s0: PF sysfs path is '/sys/devices/pci0000:00/0000:00:01.3/0000:03:00.2/0000:20:01.0/0000:22:00.0' [1749297901.811956] [prateek:19822:0] topo.c:898 UCX DEBUG /sys/class/net/enp34s0: PF sysfs path is '/sys/devices/pci0000:00/0000:00:01.3/0000:03:00.2/0000:20:01.0/0000:22:00.0' [1749297901.812082] [prateek:19822:0] topo.c:898 UCX DEBUG /sys/class/net/enp34s0: PF sysfs path is '/sys/devices/pci0000:00/0000:00:01.3/0000:03:00.2/0000:20:01.0/0000:22:00.0' [1749297901.812208] [prateek:19822:0] topo.c:898 UCX DEBUG /sys/class/net/enp34s0: PF sysfs path is '/sys/devices/pci0000:00/0000:00:01.3/0000:03:00.2/0000:20:01.0/0000:22:00.0' [1749297901.812332] [prateek:19822:0] topo.c:898 UCX DEBUG /sys/class/net/enp34s0: PF sysfs path is '/sys/devices/pci0000:00/0000:00:01.3/0000:03:00.2/0000:20:01.0/0000:22:00.0' [1749297901.812389] [prateek:19822:0] async.c:247 UCX DEBUG added async handler 0x63dfe09c9360 [id=10 ref 1] ???() to hash [1749297901.812399] [prateek:19822:0] async.c:521 UCX DEBUG listening to async event fd 10 events 0x0 mode thread_spinlock [1749297901.812411] [prateek:19822:0] sock.c:90 UCX DEBUG ioctl(req=35142, ifr_name=lo) failed: Operation not supported [1749297901.812421] [prateek:19822:0] tcp_net.c:61 UCX DEBUG speed of lo is UNKNOWN, assuming 100 Mbps [1749297901.812509] [prateek:19822:0] topo.c:895 UCX DEBUG /sys/class/net/lo: sysfs path undetected [1749297901.812516] [prateek:19822:0] topo.c:848 UCX DEBUG lo: pci bandwidth undetected, using maximal value [1749297901.812560] [prateek:19822:0] sock.c:90 UCX DEBUG ioctl(req=35142, ifr_name=lo) failed: Operation not supported [1749297901.812570] [prateek:19822:0] tcp_net.c:61 UCX DEBUG speed of lo is UNKNOWN, assuming 100 Mbps [1749297901.812632] [prateek:19822:0] topo.c:895 UCX DEBUG /sys/class/net/lo: sysfs path undetected [1749297901.812636] [prateek:19822:0] topo.c:848 UCX DEBUG lo: pci bandwidth undetected, using maximal value [1749297901.812669] [prateek:19822:0] sock.c:90 UCX DEBUG ioctl(req=35142, ifr_name=lo) failed: Operation not supported [1749297901.812678] [prateek:19822:0] tcp_net.c:61 UCX DEBUG speed of lo is UNKNOWN, assuming 100 Mbps [1749297901.812737] [prateek:19822:0] topo.c:895 UCX DEBUG /sys/class/net/lo: sysfs path undetected [1749297901.812741] [prateek:19822:0] topo.c:848 UCX DEBUG lo: pci bandwidth undetected, using maximal value [1749297901.812775] [prateek:19822:0] sock.c:90 UCX DEBUG ioctl(req=35142, ifr_name=lo) failed: Operation not supported [1749297901.812783] [prateek:19822:0] tcp_net.c:61 UCX DEBUG speed of lo is UNKNOWN, assuming 100 Mbps [1749297901.812843] [prateek:19822:0] topo.c:895 UCX DEBUG /sys/class/net/lo: sysfs path undetected [1749297901.812849] [prateek:19822:0] topo.c:848 UCX DEBUG lo: pci bandwidth undetected, using maximal value [1749297901.812903] [prateek:19822:0] sock.c:90 UCX DEBUG ioctl(req=35142, ifr_name=lo) failed: Operation not supported [1749297901.812915] [prateek:19822:0] tcp_net.c:61 UCX DEBUG speed of lo is UNKNOWN, assuming 100 Mbps [1749297901.812985] [prateek:19822:0] topo.c:895 UCX DEBUG /sys/class/net/lo: sysfs path undetected [1749297901.812992] [prateek:19822:0] topo.c:848 UCX DEBUG lo: pci bandwidth undetected, using maximal value [1749297901.813036] [prateek:19822:0] sock.c:90 UCX DEBUG ioctl(req=35142, ifr_name=lo) failed: Operation not supported [1749297901.813048] [prateek:19822:0] tcp_net.c:61 UCX DEBUG speed of lo is UNKNOWN, assuming 100 Mbps [1749297901.813124] [prateek:19822:0] topo.c:895 UCX DEBUG /sys/class/net/lo: sysfs path undetected [1749297901.813131] [prateek:19822:0] topo.c:848 UCX DEBUG lo: pci bandwidth undetected, using maximal value [1749297901.813169] [prateek:19822:0] sock.c:90 UCX DEBUG ioctl(req=35142, ifr_name=lo) failed: Operation not supported [1749297901.813180] [prateek:19822:0] tcp_net.c:61 UCX DEBUG speed of lo is UNKNOWN, assuming 100 Mbps [1749297901.813251] [prateek:19822:0] topo.c:895 UCX DEBUG /sys/class/net/lo: sysfs path undetected [1749297901.813258] [prateek:19822:0] topo.c:848 UCX DEBUG lo: pci bandwidth undetected, using maximal value [1749297901.813304] [prateek:19822:0] sock.c:90 UCX DEBUG ioctl(req=35142, ifr_name=lo) failed: Operation not supported [1749297901.813316] [prateek:19822:0] tcp_net.c:61 UCX DEBUG speed of lo is UNKNOWN, assuming 100 Mbps [1749297901.813391] [prateek:19822:0] topo.c:895 UCX DEBUG /sys/class/net/lo: sysfs path undetected [1749297901.813397] [prateek:19822:0] topo.c:848 UCX DEBUG lo: pci bandwidth undetected, using maximal value [1749297901.813445] [prateek:19822:0] sock.c:90 UCX DEBUG ioctl(req=35142, ifr_name=lo) failed: Operation not supported [1749297901.813458] [prateek:19822:0] tcp_net.c:61 UCX DEBUG speed of lo is UNKNOWN, assuming 100 Mbps [1749297901.813530] [prateek:19822:0] topo.c:895 UCX DEBUG /sys/class/net/lo: sysfs path undetected [1749297901.813536] [prateek:19822:0] topo.c:848 UCX DEBUG lo: pci bandwidth undetected, using maximal value [1749297901.813580] [prateek:19822:0] sock.c:90 UCX DEBUG ioctl(req=35142, ifr_name=lo) failed: Operation not supported [1749297901.813592] [prateek:19822:0] tcp_net.c:61 UCX DEBUG speed of lo is UNKNOWN, assuming 100 Mbps [1749297901.813664] [prateek:19822:0] topo.c:895 UCX DEBUG /sys/class/net/lo: sysfs path undetected [1749297901.813671] [prateek:19822:0] topo.c:848 UCX DEBUG lo: pci bandwidth undetected, using maximal value [1749297901.813715] [prateek:19822:0] sock.c:90 UCX DEBUG ioctl(req=35142, ifr_name=lo) failed: Operation not supported [1749297901.813727] [prateek:19822:0] tcp_net.c:61 UCX DEBUG speed of lo is UNKNOWN, assuming 100 Mbps [1749297901.813790] [prateek:19822:0] topo.c:895 UCX DEBUG /sys/class/net/lo: sysfs path undetected [1749297901.813795] [prateek:19822:0] topo.c:848 UCX DEBUG lo: pci bandwidth undetected, using maximal value [1749297901.813828] [prateek:19822:0] sock.c:90 UCX DEBUG ioctl(req=35142, ifr_name=lo) failed: Operation not supported [1749297901.813836] [prateek:19822:0] tcp_net.c:61 UCX DEBUG speed of lo is UNKNOWN, assuming 100 Mbps [1749297901.813909] [prateek:19822:0] topo.c:895 UCX DEBUG /sys/class/net/lo: sysfs path undetected [1749297901.813913] [prateek:19822:0] topo.c:848 UCX DEBUG lo: pci bandwidth undetected, using maximal value [1749297901.813946] [prateek:19822:0] sock.c:90 UCX DEBUG ioctl(req=35142, ifr_name=lo) failed: Operation not supported [1749297901.813954] [prateek:19822:0] tcp_net.c:61 UCX DEBUG speed of lo is UNKNOWN, assuming 100 Mbps [1749297901.814009] [prateek:19822:0] topo.c:895 UCX DEBUG /sys/class/net/lo: sysfs path undetected [1749297901.814014] [prateek:19822:0] topo.c:848 UCX DEBUG lo: pci bandwidth undetected, using maximal value [1749297901.814045] [prateek:19822:0] sock.c:90 UCX DEBUG ioctl(req=35142, ifr_name=lo) failed: Operation not supported [1749297901.814053] [prateek:19822:0] tcp_net.c:61 UCX DEBUG speed of lo is UNKNOWN, assuming 100 Mbps [1749297901.814109] [prateek:19822:0] topo.c:895 UCX DEBUG /sys/class/net/lo: sysfs path undetected [1749297901.814113] [prateek:19822:0] topo.c:848 UCX DEBUG lo: pci bandwidth undetected, using maximal value [1749297901.814143] [prateek:19822:0] sock.c:90 UCX DEBUG ioctl(req=35142, ifr_name=lo) failed: Operation not supported [1749297901.814151] [prateek:19822:0] tcp_net.c:61 UCX DEBUG speed of lo is UNKNOWN, assuming 100 Mbps [1749297901.814209] [prateek:19822:0] topo.c:895 UCX DEBUG /sys/class/net/lo: sysfs path undetected [1749297901.814213] [prateek:19822:0] topo.c:848 UCX DEBUG lo: pci bandwidth undetected, using maximal value [1749297901.814244] [prateek:19822:0] sock.c:90 UCX DEBUG ioctl(req=35142, ifr_name=lo) failed: Operation not supported [1749297901.814252] [prateek:19822:0] tcp_net.c:61 UCX DEBUG speed of lo is UNKNOWN, assuming 100 Mbps [1749297901.814308] [prateek:19822:0] topo.c:895 UCX DEBUG /sys/class/net/lo: sysfs path undetected [1749297901.814312] [prateek:19822:0] topo.c:848 UCX DEBUG lo: pci bandwidth undetected, using maximal value [1749297901.814342] [prateek:19822:0] sock.c:90 UCX DEBUG ioctl(req=35142, ifr_name=lo) failed: Operation not supported [1749297901.814350] [prateek:19822:0] tcp_net.c:61 UCX DEBUG speed of lo is UNKNOWN, assuming 100 Mbps [1749297901.814407] [prateek:19822:0] topo.c:895 UCX DEBUG /sys/class/net/lo: sysfs path undetected [1749297901.814411] [prateek:19822:0] topo.c:848 UCX DEBUG lo: pci bandwidth undetected, using maximal value [1749297901.814441] [prateek:19822:0] sock.c:90 UCX DEBUG ioctl(req=35142, ifr_name=lo) failed: Operation not supported [1749297901.814449] [prateek:19822:0] tcp_net.c:61 UCX DEBUG speed of lo is UNKNOWN, assuming 100 Mbps [1749297901.814506] [prateek:19822:0] topo.c:895 UCX DEBUG /sys/class/net/lo: sysfs path undetected [1749297901.814510] [prateek:19822:0] topo.c:848 UCX DEBUG lo: pci bandwidth undetected, using maximal value [1749297901.814540] [prateek:19822:0] sock.c:90 UCX DEBUG ioctl(req=35142, ifr_name=lo) failed: Operation not supported [1749297901.814548] [prateek:19822:0] tcp_net.c:61 UCX DEBUG speed of lo is UNKNOWN, assuming 100 Mbps [1749297901.814604] [prateek:19822:0] topo.c:895 UCX DEBUG /sys/class/net/lo: sysfs path undetected [1749297901.814608] [prateek:19822:0] topo.c:848 UCX DEBUG lo: pci bandwidth undetected, using maximal value [1749297901.814638] [prateek:19822:0] sock.c:90 UCX DEBUG ioctl(req=35142, ifr_name=lo) failed: Operation not supported [1749297901.814646] [prateek:19822:0] tcp_net.c:61 UCX DEBUG speed of lo is UNKNOWN, assuming 100 Mbps [1749297901.814703] [prateek:19822:0] topo.c:895 UCX DEBUG /sys/class/net/lo: sysfs path undetected [1749297901.814707] [prateek:19822:0] topo.c:848 UCX DEBUG lo: pci bandwidth undetected, using maximal value [1749297901.814737] [prateek:19822:0] sock.c:90 UCX DEBUG ioctl(req=35142, ifr_name=lo) failed: Operation not supported [1749297901.814745] [prateek:19822:0] tcp_net.c:61 UCX DEBUG speed of lo is UNKNOWN, assuming 100 Mbps [1749297901.814802] [prateek:19822:0] topo.c:895 UCX DEBUG /sys/class/net/lo: sysfs path undetected [1749297901.814806] [prateek:19822:0] topo.c:848 UCX DEBUG lo: pci bandwidth undetected, using maximal value [1749297901.814837] [prateek:19822:0] sock.c:90 UCX DEBUG ioctl(req=35142, ifr_name=lo) failed: Operation not supported [1749297901.814845] [prateek:19822:0] tcp_net.c:61 UCX DEBUG speed of lo is UNKNOWN, assuming 100 Mbps [1749297901.814907] [prateek:19822:0] topo.c:895 UCX DEBUG /sys/class/net/lo: sysfs path undetected [1749297901.814912] [prateek:19822:0] topo.c:848 UCX DEBUG lo: pci bandwidth undetected, using maximal value [1749297901.814941] [prateek:19822:0] async.c:247 UCX DEBUG added async handler 0x63dfe09f8c40 [id=12 ref 1] ???() to hash [1749297901.814952] [prateek:19822:0] async.c:521 UCX DEBUG listening to async event fd 12 events 0x0 mode thread_spinlock [1749297901.814965] [prateek:19822:0] async.c:247 UCX DEBUG added async handler 0x63dfe09f8c80 [id=14 ref 1] ???() to hash [1749297901.814971] [prateek:19822:0] async.c:521 UCX DEBUG listening to async event fd 14 events 0x0 mode thread_spinlock [1749297901.814992] [prateek:19822:0] tcp_sockcm.c:225 UCX DEBUG created tcp_sockcm 0x63dfe09f93f0 [1749297901.815001] [prateek:19822:0] mpool.c:136 UCX DEBUG mpool ucp_requests: align 64, maxelems 4294967295, elemsize 272 [1749297901.815005] [prateek:19822:0] mpool.c:136 UCX DEBUG mpool ucp_rkeys: align 64, maxelems 4294967295, elemsize 104 [1749297901.815008] [prateek:19822:0] mpool.c:136 UCX DEBUG mpool ucp_reg_bufs: align 64, maxelems 4294967295, elemsize 8216 [1749297901.815013] [prateek:19822:0] mpool.c:136 UCX DEBUG mpool ucp_am_bufs: align 64, maxelems 4294967295, elemsize 153 [1749297901.815015] [prateek:19822:0] mpool.c:136 UCX DEBUG mpool ucp_am_bufs: align 64, maxelems 4294967295, elemsize 1113 [1749297901.815019] [prateek:19822:0] mpool.c:136 UCX DEBUG mpool ucp_am_bufs: align 64, maxelems 4294967295, elemsize 65625 [1749297901.815022] [prateek:19822:0] mpool_set.c:129 UCX DEBUG mpool_set:ucp_am_bufs, sizes map 0x80000440, largest size 65536, mpools num 3 [1749297901.815047] [prateek:19822:0] parser.c:2335 UCX INFO UCX_* env variable: UCX_LOG_LEVEL=data FILE: /home/prateek/Workspace/UCX/src/common/common.cc FUNCTION : Init LINE : 163 message: Server address length : 218 bytes FILE: /home/prateek/Workspace/UCX/src/common/common.cc FUNCTION : Init LINE : 164 message: Server address : :p��_�( cuw�+2@�PO��.йY�!&��g��Q8�S�Lu*�6#��#!��a�,�����Q8S�>K��77#��I��a�,�T�Sw�+2pPw̫3O#�"��a�,�T��w�+2pPw̫3O#�nM@���a�,�TG��76$.Pw̫3�!�nM�� FILE: /home/prateek/Workspace/UCX/src/common/common.cc FUNCTION : SetSockAddr LINE : 37 message: Server will listen on 127.0.0.1:9090 [1749297901.815340] [prateek:19822:0] ucp_ep.c:405 UCX DEBUG created ep 0x74059a9e0000 to from api call [1749297901.815373] [prateek:19822:0] proto_rndv.c:514 UCX DEBUG no active message lane for tag/rndv [1749297901.815387] [prateek:19822:0] proto_rndv.c:514 UCX DEBUG no active message lane for rndv/rtr [1749297901.815391] [prateek:19822:0] proto_rndv.c:514 UCX DEBUG no active message lane for rndv/rtr/mtype [1749297901.815411] [prateek:19822:0] proto_rndv.c:514 UCX DEBUG no active message lane for rndv/ats [1749297901.815415] [prateek:19822:0] proto_rndv.c:514 UCX DEBUG no active message lane for rndv/rtr [1749297901.815421] [prateek:19822:0] proto_rndv.c:514 UCX DEBUG no active message lane for rndv/rtr [1749297901.815425] [prateek:19822:0] proto_rndv.c:514 UCX DEBUG no active message lane for rndv/rtr/mtype [1749297901.815431] [prateek:19822:0] proto_rndv.c:514 UCX DEBUG no active message lane for rndv/rtr/mtype [1749297901.815438] [prateek:19822:0] proto_rndv.c:514 UCX DEBUG no active message lane for rndv/rtr [1749297901.815442] [prateek:19822:0] proto_rndv.c:514 UCX DEBUG no active message lane for rndv/rtr/mtype [1749297901.815450] [prateek:19822:0] proto_rndv.c:514 UCX DEBUG no active message lane for rndv/rtr [1749297901.815453] [prateek:19822:0] proto_rndv.c:514 UCX DEBUG no active message lane for rndv/rtr/mtype [1749297901.815458] [prateek:19822:0] proto_rndv.c:514 UCX DEBUG no active message lane for am/rndv [1749297901.815466] [prateek:19822:0] proto_rndv.c:514 UCX DEBUG no active message lane for rndv/rtr [1749297901.815469] [prateek:19822:0] proto_rndv.c:514 UCX DEBUG no active message lane for rndv/rtr/mtype [1749297901.815473] [prateek:19822:0] proto_rndv.c:514 UCX DEBUG no active message lane for am/rndv [1749297901.815577] [prateek:19822:0] sock.c:344 UCX DEBUG connect(fd=16, src_addr=127.0.0.1:57354 dest_addr=127.0.0.1:9090): Operation now in progress [1749297901.815589] [prateek:19822:0] async.c:247 UCX DEBUG added async handler 0x63dfe0a07210 [id=16 ref 1] uct_tcp_sa_data_handler() to hash [1749297901.815597] [prateek:19822:0] async.c:521 UCX DEBUG listening to async event fd 16 events 0x2 mode thread_spinlock [1749297901.815605] [prateek:19822:0] tcp_sockcm_ep.c:923 UCX DEBUG created a TCP SOCKCM endpoint (fd=16) on tcp cm 0x63dfe09f93f0, remote addr: 127.0.0.1:9090 [1749297901.815612] [prateek:19822:0] tcp_sockcm_ep.c:1126 UCX DEBUG client created an endpoint on tcp_sockcm 0x63dfe09f93f0 id: 16 state: 2 [1749297901.815618] [prateek:19822:0] wireup_ep.c:550 UCX DEBUG ep 0x74059a9e0000: wireup_ep 0x63dfe0a06680 set next_ep 0x63dfe0a06970 FILE: /home/prateek/Workspace/UCX/src/client/client.cc FUNCTION : Client LINE : 13 message: Client created with server address: 127.0.0.1 and port: 9090 FILE: /home/prateek/Workspace/UCX/src/client/client.cc FUNCTION : SendAcvtiveMsg LINE : 92 message: ENTRY [1749297901.815658] [prateek:19822:0] mpool.c:281 UCX DEBUG mpool ucp_requests: allocated chunk 0x63dfe0a075e4 of 41044 bytes with 128 elements [1749297901.815764] [prateek:19822:0] sock.c:1076 UCX DEBUG check ifname for socket on 127.0.0.1:0 [1749297901.815943] [prateek:19822:0] sock.c:1094 UCX DEBUG matching ip found iface on lo [1749297901.815952] [prateek:19822:0] wireup_cm.c:607 UCX DEBUG client created ep 0x74059a9e0000 on device lo, tl_bitmap 0x4 0x0 on cm tcp [1749297901.815960] [prateek:19822:0] wireup_cm.c:184 UCX DEBUG ep 0x74059a9e0000: init_flags 0x36 [1749297901.816001] [prateek:19822:0] wireup_cm.c:128 UCX DIAG client ep 0x74059a9e0000 failed to connect to 127.0.0.1:9090 using tcp cms [1749297901.816007] [prateek:19822:0] ucp_ep.c:1474 UCX DEBUG ep 0x74059a9e0000: set_ep_failed status Destination is unreachable on lane[0]=0x63dfe0a06680 [1749297901.816011] [prateek:19822:0] ucp_ep.c:1437 UCX DEBUG ep 0x74059a9e0000: discarding lanes [1749297901.816016] [prateek:19822:0] ucp_ep.c:1445 UCX DEBUG ep 0x74059a9e0000: discard uct_ep[0]=0x63dfe0a06680 [1749297901.816021] [prateek:19822:0] proto_common.c:824 UCX DEBUG abort request 0x63dfe0a11500 proto reconfig status Destination is unreachable [1749297901.816029] [prateek:19822:0] wireup_ep.c:440 UCX DEBUG ep 0x74059a9e0000: destroy wireup ep 0x63dfe0a06680 [1749297901.816036] [prateek:19822:0] ucp_ep.c:3505 UCX DEBUG ep 0x74059a9e0000: calling user error callback 0x63dfba0f0cde with arg 0x7fff7e44b3f0 and status Destination is unreachable FILE: /home/prateek/Workspace/UCX/src/client/client.cc FUNCTION : ErrorHandlingCb LINE : 65 message: Error handling callback invoked with status: Destination is unreachable FILE: /home/prateek/Workspace/UCX/src/client/client.cc FUNCTION : SendAcvtiveMsg LINE : 112 message: EXIT [1749297901.816174] [prateek:19822:0] async.c:714 UCX DIAG async handler table is not empty during exit (contains 8 elems) [1749297901.816182] [prateek:19822:0] thread.c:433 UCX DIAG async thread still running (use count 8) [1749297901.816185] [prateek:19822:0] thread.c:433 UCX DIAG async thread still running (use count 8)