diff --git a/.gitignore b/.gitignore index aabc20d7..d1a47d26 100644 --- a/.gitignore +++ b/.gitignore @@ -219,3 +219,8 @@ assets/* # Local Labtasker state and attempt journals .labtasker/ + +# Sharpa WebXR sim eval outputs and venv +.sharpa_sim_eval/ +.venv_webxr/ +MUJOCO_LOG.TXT diff --git a/.gitmodules b/.gitmodules index f807f471..62f96438 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,9 @@ [submodule "third_party/cosmos-predict2.5"] path = third_party/cosmos-predict2.5 url = https://github.com/nvidia-cosmos/cosmos-predict2.5.git +[submodule "third_party/giga-world-policy"] + path = third_party/giga-world-policy + url = https://github.com/open-gigaai/giga-world-policy.git +[submodule "third_party/WebXR-Teleop"] + path = third_party/WebXR-Teleop + url = git@github.com:chengyuan-qian-applovin/WebXR-Teleop.git diff --git a/assets/giga_sharpa_train_job/prep_eef_full_v1/norm_stats.json b/assets/giga_sharpa_train_job/prep_eef_full_v1/norm_stats.json new file mode 100644 index 00000000..d0173dc5 --- /dev/null +++ b/assets/giga_sharpa_train_job/prep_eef_full_v1/norm_stats.json @@ -0,0 +1,861 @@ +{ + "norm_stats": { + "observation.state": { + "mean": [ + -0.24161164598017557, + 0.03136722078482579, + 0.5087595909129758, + 0.1325137530163079, + 0.4516462793798748, + 0.5220324096274115, + -0.6431803347468297, + 0.008208175913760156, + 0.15713931542416668, + 0.15996516882696607, + -0.05616770516101251, + 0.49418502387325186, + -0.20789234642371632, + 0.5842118729976427, + 0.5269730482516803, + -0.7228246380936525, + 0.03020686753874721, + -0.3098442746071791, + 0.44895951290144015, + -0.027803426641668856, + 0.09811883251213434, + 0.018498015402749476, + 0.09828113474553558, + 0.24626102229117613, + -0.006156810250440333, + 0.18557585401994564, + 0.070415718273613, + 0.31731170815779064, + 0.006889170911264003, + 0.15843104055467996, + 0.10392265093253987, + 0.2888860384042701, + 0.007902179020756073, + 0.2054776761564498, + 0.11271095310653192, + 0.04856232038205333, + 0.22249027283391137, + 0.03804580991892623, + 0.2199403501605057, + 0.1446221240637349, + 1.1428144557034243, + -0.056123438425254246, + 0.008320059887187406, + 0.051942423968418125, + 0.25024238837852236, + 0.5890936444311465, + 0.06397967781171553, + 0.273970240109811, + 0.14642991152532314, + 0.6632051945357522, + 0.08227494225841654, + 0.2962236495022631, + 0.15683075000175123, + 0.7460098606513252, + 0.06237153204055145, + 0.7943566533339249, + 0.5246287987522752, + 0.058601376596885336, + 0.7283723919966256, + 0.09355260387567463, + 0.7973314496576207, + 0.5675430463088481 + ], + "std": [ + 0.13082294397035962, + 0.08620928735393311, + 0.06597394012447153, + 0.29890766657272266, + 0.38294222078992346, + 0.5195645926866144, + 0.4619170922403358, + 0.4655752630893641, + 0.3625344102254094, + 0.11438536255706665, + 0.08715031579888338, + 0.11082528723827831, + 0.36871913697263364, + 0.34621498065693146, + 0.28628316603998766, + 0.37704223419232613, + 0.37422502992016415, + 0.3136931531874326, + 0.6807424777380108, + 0.1244689679491296, + 0.42769594164467944, + 0.14463082180518247, + 0.24092705709387685, + 0.4213709907271711, + 0.10969057997180214, + 0.3123718498512177, + 0.1703020446689459, + 0.4707706577088732, + 0.09702341582299623, + 0.30286873281083854, + 0.1967318167363677, + 0.46467865131817143, + 0.0858612858445945, + 0.3552841193735789, + 0.21996520222054775, + 0.0814441497608117, + 0.39392218164811377, + 0.1096525483609896, + 0.36018324378501737, + 0.27896622353110817, + 0.792690438066716, + 0.1579677352294934, + 0.5673408307450379, + 0.21669078310770962, + 0.360463522170216, + 0.4376015644526251, + 0.1534413181542209, + 0.3061373994337979, + 0.3020058413224959, + 0.4041790065117659, + 0.11694093572059691, + 0.4240472627708225, + 0.3284201796905436, + 0.517800534037011, + 0.1171390083886638, + 0.6224126529941707, + 0.5196322170692522, + 0.06737685320548688, + 0.5461113898854449, + 0.13331798938358164, + 0.6217355046381302, + 0.5012203921036961 + ], + "min": [ + -0.639397919178009, + -0.36186373233795166, + -0.22894780337810516, + -0.7038918733596802, + -0.9928406476974487, + -0.9905152320861816, + -1.0, + -0.9853591918945312, + -0.8998257517814636, + -0.2256472408771515, + -0.3427990972995758, + 0.06608644872903824, + -1.0, + -0.9868537783622742, + -0.9993075728416443, + -1.0, + -0.9999626874923706, + -0.9999516010284424, + -0.6103947162628174, + -0.7394280433654785, + -0.5751699209213257, + -0.7548040151596069, + -1.4215848445892334, + -0.8330821394920349, + -0.7341446876525879, + -1.8892686367034912, + -1.3816496133804321, + -0.6174737215042114, + -0.4878499209880829, + -1.3443012237548828, + -1.3784130811691284, + -0.7759770750999451, + -0.32079213857650757, + -1.488852620124817, + -0.9112948179244995, + -0.49267587065696716, + -0.5136713981628418, + -0.3484576940536499, + -0.6739005446434021, + -0.5496805310249329, + -0.39951249957084656, + -1.080626368522644, + -0.6465205550193787, + -0.48719561100006104, + -0.6214686632156372, + -0.5278289914131165, + -0.7475773096084595, + -1.3171029090881348, + -1.5782225131988525, + -0.6566876769065857, + -0.4371297359466553, + -1.7980835437774658, + -2.7197554111480713, + -0.7194892764091492, + -0.3161975145339966, + -1.3625333309173584, + -1.6966453790664673, + -0.497658371925354, + -0.47061070799827576, + -0.34944450855255127, + -0.49938562512397766, + -0.8870709538459778 + ], + "max": [ + 0.19668936729431152, + 0.3940102458000183, + 0.7924806475639343, + 1.0, + 0.9996849298477173, + 0.9999736547470093, + 0.9996882677078247, + 0.99998539686203, + 0.9999938607215881, + 0.6947617530822754, + 0.3566063344478607, + 0.8029068112373352, + 0.8683165311813354, + 0.9999832510948181, + 0.9999539852142334, + 0.9991872310638428, + 0.9992381930351257, + 0.776995062828064, + 2.330277919769287, + 0.4544250965118408, + 1.3316869735717773, + 0.3792952299118042, + 2.7403805255889893, + 1.5672926902770996, + 0.35662561655044556, + 1.9258087873458862, + 1.7055160999298096, + 1.6262203454971313, + 0.4344328045845032, + 2.209760904312134, + 2.0428123474121094, + 1.5730489492416382, + 0.4506634771823883, + 1.8550935983657837, + 1.8741395473480225, + 0.7153012752532959, + 1.6161402463912964, + 0.8369439840316772, + 2.0515403747558594, + 2.2719221115112305, + 2.244424819946289, + 0.4166914224624634, + 1.3248519897460938, + 0.5174813270568848, + 3.0555050373077393, + 1.600819706916809, + 0.35386231541633606, + 2.7688395977020264, + 2.842449426651001, + 1.592158317565918, + 0.4764637351036072, + 2.3524322509765625, + 1.7978993654251099, + 1.582544207572937, + 0.4663342237472534, + 2.5466439723968506, + 1.8985177278518677, + 0.705226480960846, + 1.6191809177398682, + 0.7089475393295288, + 2.2493317127227783, + 2.400118350982666 + ], + "q01": [ + -0.5089566707611084, + -0.20606261491775513, + 0.33828097581863403, + -0.1352740377187729, + -0.573979914188385, + -0.8681278228759766, + -0.9994858503341675, + -0.7370920777320862, + -0.49688559770584106, + -0.0908486396074295, + -0.2811453938484192, + 0.15891443192958832, + -0.9999259114265442, + -0.8035138249397278, + -0.3030959665775299, + -0.9996259808540344, + -0.849103569984436, + -0.9185089468955994, + -0.17304107546806335, + -0.35120218992233276, + -0.5235990285873413, + -0.34970566630363464, + -0.008934928104281425, + -0.1745348423719406, + -0.31275877356529236, + -0.008514909073710442, + -0.0033223580103367567, + -0.1340133100748062, + -0.24535249173641205, + -0.0027147468645125628, + -0.002551741199567914, + -0.17452536523342133, + -0.23413261771202087, + -0.001960749737918377, + -0.0030052007641643286, + -0.00046919280430302024, + -0.1745452880859375, + -0.21563035249710083, + -8.72170930961147e-05, + -0.004510961472988129, + -0.07948853820562363, + -0.36095690727233887, + -0.5236169695854187, + -0.3491171598434448, + -0.011365680955350399, + -0.1745273321866989, + -0.2804180979728699, + -0.019906865432858467, + -0.008482543751597404, + -0.01592474803328514, + -0.12183884531259537, + -0.023174645379185677, + -0.00920023676007986, + -0.17361119389533997, + -0.10587168484926224, + -0.0003209643764421344, + -0.003711037104949355, + -0.02733287215232849, + -0.17453639209270477, + -0.040462568402290344, + -8.55898906593211e-05, + -0.0013069849228486419 + ], + "q99": [ + 0.031235402449965477, + 0.16854926943778992, + 0.6469495892524719, + 0.9988163113594055, + 0.8657280802726746, + 0.9026405215263367, + 0.5313512086868286, + 0.8442007899284363, + 0.8664464950561523, + 0.42513877153396606, + 0.11358777433633804, + 0.6794008016586304, + 0.5416536927223206, + 0.8969926834106445, + 0.9217153191566467, + 0.7517461180686951, + 0.8513458371162415, + 0.40079212188720703, + 1.9052537679672241, + 0.17886169254779816, + 1.276299238204956, + 0.3491009473800659, + 1.1032413244247437, + 1.347165584564209, + 0.33174630999565125, + 1.2038910388946533, + 0.8124955892562866, + 1.4666558504104614, + 0.3414551019668579, + 1.304782509803772, + 0.866578996181488, + 1.473488688468933, + 0.2932700514793396, + 1.4999181032180786, + 1.0602296590805054, + 0.26402977108955383, + 1.4704512357711792, + 0.3490976095199585, + 1.5332039594650269, + 1.2398247718811035, + 1.9200338125228882, + 0.21106809377670288, + 1.218489646911621, + 0.3493114709854126, + 1.3280563354492188, + 1.3095539808273315, + 0.3427296280860901, + 1.1740342378616333, + 1.2881155014038086, + 1.3887299299240112, + 0.34922733902931213, + 1.6419121026992798, + 1.292904257774353, + 1.467363715171814, + 0.3490672707557678, + 1.7031532526016235, + 1.2131024599075317, + 0.26179227232933044, + 1.5464959144592285, + 0.34910258650779724, + 1.601161003112793, + 1.295600414276123 + ] + }, + "action": { + "mean": [ + 8.891515813136553e-05, + -0.0007282664445308291, + 0.001712280439138051, + 0.9896946978986759, + -0.012058683257291553, + -0.0004412582871225987, + 0.010853493085710757, + 0.9884894947680596, + -0.00651699996909449, + -0.0015880902654774997, + 0.0022033952503273144, + 0.0019359964041165008, + 0.9835448118502648, + 0.008760268571646287, + 0.001202746801430436, + -0.008113634358484324, + 0.9799888067184267, + -0.00029789057374178876, + 0.47020610977306054, + -0.02382507393091464, + 0.10902949262201612, + 0.020647581781090527, + 0.10283875855842689, + 0.2621404659965415, + -0.004842815447233024, + 0.19971190510217038, + 0.07629136915672986, + 0.33647264618898376, + 0.006777437453082529, + 0.17586997542108976, + 0.11130612368960927, + 0.30638022652702424, + 0.006924002710421001, + 0.22117986922132543, + 0.11844569844160123, + 0.05214437642592946, + 0.23651444798873106, + 0.03933825596946899, + 0.2362252108426182, + 0.14651338086230906, + 1.1942055869831183, + -0.028020893425616096, + 0.021874885634794086, + 0.05269668263447357, + 0.25040790033657756, + 0.613666392467174, + 0.12405627197513526, + 0.31935359231992394, + 0.18489469567112593, + 0.7019398940442269, + 0.03830860821065096, + 0.3310093835032086, + 0.1900796465092799, + 0.7847665810042841, + 0.06445381974305833, + 0.8231501406170114, + 0.5629582066058405, + 0.06617598699252494, + 0.7560827644144651, + 0.09672881361112978, + 0.8455007162335094, + 0.596644107789853 + ], + "std": [ + 0.0242008260854784, + 0.022553150980192213, + 0.027627874408218633, + 0.06872952931228773, + 0.10194872423409927, + 0.072398263818877, + 0.10329741559372263, + 0.07490092881255205, + 0.08029785232188856, + 0.033538111376492666, + 0.03288337247816609, + 0.03894130108945881, + 0.08894363453760008, + 0.13351600165761376, + 0.08260704042241084, + 0.13332498556096292, + 0.10275633289043554, + 0.10593207198798672, + 0.6934037752494657, + 0.1252815018565069, + 0.44545082134238145, + 0.14398125591416627, + 0.24456746217587758, + 0.434931390108131, + 0.11469550266935732, + 0.33322605887378914, + 0.1804421473316076, + 0.4843771506093775, + 0.09860135198088342, + 0.33110965232187267, + 0.20744075451741809, + 0.47804831203135173, + 0.089044557584893, + 0.37395459904985107, + 0.2278474601401687, + 0.08362555091659496, + 0.40469735743507085, + 0.11242232896356844, + 0.3731889986584222, + 0.27659459427618405, + 0.7846653806135508, + 0.1675061765504528, + 0.5846871282427512, + 0.21865499012711695, + 0.34368720539876363, + 0.4300928244881982, + 0.20378966557449227, + 0.34544187131814025, + 0.329426048755819, + 0.40811866898547544, + 0.13500944156296982, + 0.4470490507940994, + 0.3445578476028872, + 0.5194916723827201, + 0.12092095504615319, + 0.5925754173815058, + 0.5085901151507449, + 0.0689010118680918, + 0.5355658690656181, + 0.13679453694895907, + 0.6092452270319102, + 0.4865772730502878 + ], + "min": [ + -0.6523548364639282, + -0.5554970502853394, + -0.67133629322052, + -0.9994788765907288, + -0.9999980926513672, + -0.9982020258903503, + -0.9999882578849792, + -0.9911317229270935, + -0.9994003772735596, + -0.522659182548523, + -0.4864750802516937, + -0.6121780276298523, + -0.9970647692680359, + -0.9999944567680359, + -0.9999762177467346, + -1.0, + -0.9999954104423523, + -0.9999896883964539, + -0.1745000034570694, + -0.3490999937057495, + -0.5235999822616577, + -0.3490999937057495, + 0.0, + -0.1745329350233078, + -0.3490999937057495, + 0.0, + 0.0, + -0.1745329350233078, + -0.3490990102291107, + 0.0, + 0.0, + -0.1745329350233078, + -0.3490990102291107, + 0.0, + 0.0, + 0.0, + -0.1745329350233078, + -0.3490999937057495, + 0.0, + 0.0, + -0.1745000034570694, + -0.3490999937057495, + -0.5235999822616577, + -0.3490999937057495, + -2.7755575615628914e-17, + -0.1745329350233078, + -0.3490999937057495, + -2.7755575615628914e-17, + 0.0, + -0.1745329350233078, + -0.3490999937057495, + -1.1102230246251565e-15, + 0.0, + -0.1745329350233078, + -0.3490999937057495, + -2.220446049250313e-15, + 0.0, + 0.0, + -0.1745329350233078, + -0.3490999937057495, + -2.220446049250313e-15, + 0.0 + ], + "max": [ + 0.4831361472606659, + 0.45950591564178467, + 0.4451819360256195, + 1.0, + 0.9999938607215881, + 0.999969482421875, + 1.0, + 1.0, + 0.9904471635818481, + 0.4447639286518097, + 0.68703293800354, + 0.5195640325546265, + 1.0, + 0.9999996423721313, + 0.9918366074562073, + 0.9999994039535522, + 1.0, + 0.9999847412109375, + 1.9198999404907227, + 0.3490999937057495, + 1.396299958229065, + 0.3490999937057495, + 1.745300054550171, + 1.5707999467849731, + 0.34646233916282654, + 1.745300054550171, + 1.396299958229065, + 1.5707999467849731, + 0.3490999937057495, + 1.745300054550171, + 1.396299958229065, + 1.5707999467849731, + 0.3490999937057495, + 1.745300054550171, + 1.396299958229065, + 0.26179999113082886, + 1.5707999467849731, + 0.3490999937057495, + 1.745300054550171, + 1.396299958229065, + 1.9198999404907227, + 0.3490999937057495, + 1.396299958229065, + 0.3490999937057495, + 1.745300054550171, + 1.5707999467849731, + 0.3490999937057495, + 1.745300054550171, + 1.396299958229065, + 1.5707999467849731, + 0.3490999937057495, + 1.745300054550171, + 1.396299958229065, + 1.5707999467849731, + 0.3490999937057495, + 1.745300054550171, + 1.396299958229065, + 0.26179999113082886, + 1.5707999467849731, + 0.3490999937057495, + 1.745300054550171, + 1.396299958229065 + ], + "q01": [ + -0.09976901113986969, + -0.0920414924621582, + -0.08828199654817581, + 0.745243489742279, + -0.4655064642429352, + -0.2249973714351654, + -0.2500058710575104, + 0.7127324342727661, + -0.3348027765750885, + -0.11656700074672699, + -0.10475271940231323, + -0.10623228549957275, + 0.5712411403656006, + -0.3554556369781494, + -0.2579931616783142, + -0.6856547594070435, + 0.5040917992591858, + -0.3962602913379669, + -0.1744769960641861, + -0.3490999937057495, + -0.5235990285873413, + -0.3490990102291107, + 0.0, + -0.1745329350233078, + -0.31150540709495544, + 0.0, + 0.0, + -0.12921439111232758, + -0.3398153483867645, + 0.0, + 0.0, + -0.1745329350233078, + -0.3070765733718872, + 0.0, + 0.0, + 0.0, + -0.1745329350233078, + -0.28349024057388306, + 0.0, + 0.0, + -0.07308129221200943, + -0.3490999937057495, + -0.5235990285873413, + -0.3490990102291107, + 0.0, + -0.1745329350233078, + -0.27519360184669495, + 0.0, + 0.0, + -0.0025081795174628496, + -0.12191461026668549, + 0.0, + 0.0, + -0.17451100051403046, + -0.11820222437381744, + 0.0, + 0.0, + 0.0, + -0.1745329350233078, + -0.03797176852822304, + 0.0, + 0.0 + ], + "q99": [ + 0.08423396944999695, + 0.0763106644153595, + 0.11273297667503357, + 1.0, + 0.21702365577220917, + 0.3187008798122406, + 0.4614938497543335, + 1.0, + 0.2509491741657257, + 0.09711240977048874, + 0.13243594765663147, + 0.16566134989261627, + 1.0, + 0.6979382038116455, + 0.3348003625869751, + 0.35824066400527954, + 1.0, + 0.4161249101161957, + 1.9198989868164062, + 0.21230100095272064, + 1.3876873254776, + 0.3490999937057495, + 1.103253722190857, + 1.3532395362854004, + 0.34646177291870117, + 1.3411468267440796, + 0.8568006753921509, + 1.511929988861084, + 0.33006319403648376, + 1.4297751188278198, + 0.9465121626853943, + 1.5705009698867798, + 0.29327067732810974, + 1.5, + 1.1106593608856201, + 0.26179999113082886, + 1.5099327564239502, + 0.3490976393222809, + 1.6074775457382202, + 1.227154016494751, + 1.9198989868164062, + 0.21674613654613495, + 1.2944554090499878, + 0.3490999937057495, + 1.0998455286026, + 1.4071946144104004, + 0.3490990102291107, + 1.5587736368179321, + 1.2950729131698608, + 1.4689702987670898, + 0.34909895062446594, + 1.6419082880020142, + 1.2930879592895508, + 1.5707999467849731, + 0.3483070433139801, + 1.600000023841858, + 1.2108842134475708, + 0.26179999113082886, + 1.5707999467849731, + 0.3490990102291107, + 1.6674644947052002, + 1.285607099533081 + ] + } + }, + "metadata": { + "split": "train", + "datasets": [ + "assets/sharpa_full_meta/gpt_fleet_depth", + "assets/sharpa_full_meta/simteleop_1004_fleet" + ], + "action_repr": "eef", + "codec": "giga", + "rot_format": "rot6d", + "reference": "current_state", + "channel_names": [ + "left_wrist_x", + "left_wrist_y", + "left_wrist_z", + "left_wrist_r11", + "left_wrist_r21", + "left_wrist_r31", + "left_wrist_r12", + "left_wrist_r22", + "left_wrist_r32", + "right_wrist_x", + "right_wrist_y", + "right_wrist_z", + "right_wrist_r11", + "right_wrist_r21", + "right_wrist_r31", + "right_wrist_r12", + "right_wrist_r22", + "right_wrist_r32", + "left_thumb_CMC_FE", + "left_thumb_CMC_AA", + "left_thumb_MCP_FE", + "left_thumb_MCP_AA", + "left_thumb_IP", + "left_index_MCP_FE", + "left_index_MCP_AA", + "left_index_PIP", + "left_index_DIP", + "left_middle_MCP_FE", + "left_middle_MCP_AA", + "left_middle_PIP", + "left_middle_DIP", + "left_ring_MCP_FE", + "left_ring_MCP_AA", + "left_ring_PIP", + "left_ring_DIP", + "left_pinky_CMC", + "left_pinky_MCP_FE", + "left_pinky_MCP_AA", + "left_pinky_PIP", + "left_pinky_DIP", + "right_thumb_CMC_FE", + "right_thumb_CMC_AA", + "right_thumb_MCP_FE", + "right_thumb_MCP_AA", + "right_thumb_IP", + "right_index_MCP_FE", + "right_index_MCP_AA", + "right_index_PIP", + "right_index_DIP", + "right_middle_MCP_FE", + "right_middle_MCP_AA", + "right_middle_PIP", + "right_middle_DIP", + "right_ring_MCP_FE", + "right_ring_MCP_AA", + "right_ring_PIP", + "right_ring_DIP", + "right_pinky_CMC", + "right_pinky_MCP_FE", + "right_pinky_MCP_AA", + "right_pinky_PIP", + "right_pinky_DIP" + ], + "num_episodes": 8806, + "num_state_samples": 3791451, + "num_action_samples": 121326432, + "action_horizon": 32, + "quantile_sample_limit": 2000000, + "widened_degenerate_dims": { + "observation.state": [], + "action": [] + }, + "split_file": "assets/sharpa_full_meta/split_v1.json" + } +} \ No newline at end of file diff --git a/assets/giga_sharpa_train_job/prep_eef_full_v1/t5_task_embeddings.pt b/assets/giga_sharpa_train_job/prep_eef_full_v1/t5_task_embeddings.pt new file mode 100644 index 00000000..3766cc4d Binary files /dev/null and b/assets/giga_sharpa_train_job/prep_eef_full_v1/t5_task_embeddings.pt differ diff --git a/assets/robot_descriptions/SOURCE.txt b/assets/robot_descriptions/SOURCE.txt new file mode 100644 index 00000000..91d63627 --- /dev/null +++ b/assets/robot_descriptions/SOURCE.txt @@ -0,0 +1 @@ +Copied from github.com/chengyuan-qian-applovin/WebXR-Teleop assets/{sharpa,yam_ultra} @ ecc0468c591366f865ed4babd8085267b7fc8fec; reference_rollout_gpt_dataset.npz is one MuJoCo rollout from gs://foundational-research/yjw/gpt_dataset (used to validate FK + joint order). diff --git a/assets/robot_descriptions/SOURCE_COMMIT.txt b/assets/robot_descriptions/SOURCE_COMMIT.txt new file mode 100644 index 00000000..19419c7e --- /dev/null +++ b/assets/robot_descriptions/SOURCE_COMMIT.txt @@ -0,0 +1 @@ +ecc0468c591366f865ed4babd8085267b7fc8fec diff --git a/assets/robot_descriptions/reference_rollout_gpt_dataset.npz b/assets/robot_descriptions/reference_rollout_gpt_dataset.npz new file mode 100644 index 00000000..8c72f9e8 Binary files /dev/null and b/assets/robot_descriptions/reference_rollout_gpt_dataset.npz differ diff --git a/assets/robot_descriptions/sharpa_wave/left_sharpa_wave_with_flange.urdf b/assets/robot_descriptions/sharpa_wave/left_sharpa_wave_with_flange.urdf new file mode 100644 index 00000000..29db8d21 --- /dev/null +++ b/assets/robot_descriptions/sharpa_wave/left_sharpa_wave_with_flange.urdf @@ -0,0 +1,1538 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/assets/robot_descriptions/sharpa_wave/right_sharpa_wave_with_flange.urdf b/assets/robot_descriptions/sharpa_wave/right_sharpa_wave_with_flange.urdf new file mode 100644 index 00000000..457b22d3 --- /dev/null +++ b/assets/robot_descriptions/sharpa_wave/right_sharpa_wave_with_flange.urdf @@ -0,0 +1,1537 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/assets/robot_descriptions/yam_ultra/left_yam.urdf b/assets/robot_descriptions/yam_ultra/left_yam.urdf new file mode 100644 index 00000000..75b7fd8e --- /dev/null +++ b/assets/robot_descriptions/yam_ultra/left_yam.urdf @@ -0,0 +1,154 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/assets/robot_descriptions/yam_ultra/right_yam.urdf b/assets/robot_descriptions/yam_ultra/right_yam.urdf new file mode 100644 index 00000000..b6dbd627 --- /dev/null +++ b/assets/robot_descriptions/yam_ultra/right_yam.urdf @@ -0,0 +1,154 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/benchmarks/sharpa/__init__.py b/benchmarks/sharpa/__init__.py new file mode 100644 index 00000000..f67a5f4a --- /dev/null +++ b/benchmarks/sharpa/__init__.py @@ -0,0 +1 @@ +"""Deterministic evaluation for the WebXR YAM Ultra / Sharpa scenes.""" diff --git a/benchmarks/sharpa/verifiers.py b/benchmarks/sharpa/verifiers.py new file mode 100644 index 00000000..2b25c2ea --- /dev/null +++ b/benchmarks/sharpa/verifiers.py @@ -0,0 +1,203 @@ +"""State-only task checks shared by offline audits and live MuJoCo rollouts. + +Thresholds are provisional physical criteria, not human-calibrated success labels. +All rotations are wxyz. Success requires a continuous terminal stability window. +""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass + +import numpy as np +from scipy.spatial.transform import Rotation + +LAPTOP_JOINT = "articraft_rec_laptop_clamshell_0002_joints_root_to_lid" +LAPTOP_BASE = "world_laptop_03" +LAPTOP_LID = "articraft_rec_laptop_clamshell_0002_lid" +SCENES = { + "close_laptop": "00082_GigaHands_p007-laptop_043_seg000", + "box_lid": "00023_OakInk-v2_scene_02__A001_seq__4330d8d30c1293560d96__2023-04-", +} +PROMPTS = {"close_laptop": "Close the laptop lid gently", "box_lid": "Put on the box lid."} + + +def rotation(quat): + q = np.asarray(quat, dtype=float) + if q.shape != (4,) or not np.isfinite(q).all() or abs(np.linalg.norm(q) - 1) > 0.01: + raise ValueError("Expected a normalized finite wxyz quaternion") + return Rotation.from_quat(q[[1, 2, 3, 0]]).as_matrix() + + +@dataclass +class Snapshot: + time: float + positions: dict[str, np.ndarray] + quaternions: dict[str, np.ndarray] + joints: dict[str, float] + contacts: set[tuple[str, str]] + + def hand_contact(self, body): + return any( + (a == body and b.startswith(("left_", "right_"))) or (b == body and a.startswith(("left_", "right_"))) + for a, b in self.contacts + ) + + def touching(self, a, b): + return (a, b) in self.contacts or (b, a) in self.contacts + + +@dataclass(frozen=True) +class Thresholds: + hold_s: float = 0.5 + max_sample_gap_s: float = 0.075 + # Asset lid plane is rotated -112 degrees at q=0: q=+112 closes it. + laptop_closed_rad: float = np.deg2rad(112) + laptop_tolerance_rad: float = np.deg2rad(6) + laptop_min_motion_rad: float = np.deg2rad(10) + laptop_max_speed_rad_s: float = 0.2 + laptop_max_closing_speed_rad_s: float = 1.0 + box_target_height_m: float = 0.078 + box_xy_tolerance_m: float = 0.015 + box_height_tolerance_m: float = 0.012 + box_tilt_tolerance_rad: float = np.deg2rad(10) + box_max_linear_speed_m_s: float = 0.025 + box_max_angular_speed_rad_s: float = 0.2 + box_min_motion_m: float = 0.05 + + def __post_init__(self): + for name, value in asdict(self).items(): + if not np.isfinite(value) or value <= 0: + raise ValueError(f"{name} must be finite and positive") + + +class Verifier: + def __init__(self, task: str, thresholds: Thresholds | None = None): + if task not in SCENES: + raise ValueError(f"Unsupported task {task!r}; supported: {list(SCENES)}") + self.task = task + self.thresholds = thresholds or Thresholds() + self.initial = self.previous = None + self.stable_since = None + self.interacted = False + self.peak_closing_speed = 0.0 + self.checks = {} + self.metrics = {} + + def reset(self, initial: Snapshot): + self.__init__(self.task, self.thresholds) + self._validate(initial) + self.initial = self.previous = initial + + def _validate(self, state): + bodies = [LAPTOP_BASE, LAPTOP_LID] if self.task == "close_laptop" else ["world_box", "world_lid_09"] + if not np.isfinite(state.time): + raise ValueError("Non-finite timestamp") + for name in bodies: + p = np.asarray(state.positions[name]) + if p.shape != (3,) or not np.isfinite(p).all(): + raise ValueError(f"Invalid position for {name}") + rotation(state.quaternions[name]) + if self.task == "close_laptop" and not np.isfinite(state.joints[LAPTOP_JOINT]): + raise ValueError("Non-finite laptop joint") + + def update(self, state: Snapshot): + if self.initial is None: + raise RuntimeError("Call reset(initial_state) before update") + self._validate(state) + dt = state.time - self.previous.time + if dt <= 0 or dt > self.thresholds.max_sample_gap_s + 1e-9: + raise ValueError("Samples must have strictly increasing timestamps without large gaps") + if self.task == "close_laptop": + self._laptop(state, dt) + else: + self._box(state, dt) + if all(self.checks.values()): + if self.stable_since is None: + self.stable_since = state.time + else: + self.stable_since = None + self.previous = state + return self.finalize() + + def _laptop(self, s, dt): + t = self.thresholds + q = s.joints[LAPTOP_JOINT] + speed = (q - self.previous.joints[LAPTOP_JOINT]) / dt + error = abs(q - t.laptop_closed_rad) + self.interacted |= s.hand_contact(LAPTOP_LID) + # Only motion approaching the closed pose counts towards the gentle-close guard. + old_error = abs(self.previous.joints[LAPTOP_JOINT] - t.laptop_closed_rad) + if error < old_error: + self.peak_closing_speed = max(self.peak_closing_speed, abs(speed)) + initial_error = abs(self.initial.joints[LAPTOP_JOINT] - t.laptop_closed_rad) + self.checks = { + "started_open": initial_error > t.laptop_tolerance_rad, + "hand_interacted": self.interacted, + "closed": error <= t.laptop_tolerance_rad, + "made_progress": initial_error - error >= t.laptop_min_motion_rad, + "joint_stable": abs(speed) <= t.laptop_max_speed_rad_s, + "gentle_motion": self.peak_closing_speed <= t.laptop_max_closing_speed_rad_s, + "base_upright": rotation(s.quaternions[LAPTOP_BASE])[2, 2] >= np.cos(np.deg2rad(15)), + "base_not_dropped": s.positions[LAPTOP_BASE][2] >= self.initial.positions[LAPTOP_BASE][2] - 0.06, + } + self.metrics = { + "angle_rad": q, + "angle_error_rad": error, + "speed_rad_s": speed, + "peak_closing_speed_rad_s": self.peak_closing_speed, + } + + def _box(self, s, dt): + t = self.thresholds + box, lid = "world_box", "world_lid_09" + rb, rl = rotation(s.quaternions[box]), rotation(s.quaternions[lid]) + relative = rb.T @ (s.positions[lid] - s.positions[box]) + tilt = np.arccos(np.clip(abs((rb.T @ rl)[2, 2]), 0, 1)) + speed = max(np.linalg.norm(s.positions[b] - self.previous.positions[b]) / dt for b in [box, lid]) + angular_speed = max( + Rotation.from_matrix(rotation(self.previous.quaternions[b]).T @ rotation(s.quaternions[b])).magnitude() / dt + for b in [box, lid] + ) + motion = np.linalg.norm(s.positions[lid] - self.initial.positions[lid]) + initial_relative = rotation(self.initial.quaternions[box]).T @ ( + self.initial.positions[lid] - self.initial.positions[box] + ) + self.interacted |= s.hand_contact(lid) + self.checks = { + "started_separate": np.linalg.norm(initial_relative[:2]) > t.box_xy_tolerance_m, + "hand_interacted": self.interacted, + "lid_moved": motion >= t.box_min_motion_m, + "xy_aligned": np.linalg.norm(relative[:2]) <= t.box_xy_tolerance_m, + "height_aligned": abs(relative[2] - t.box_target_height_m) <= t.box_height_tolerance_m, + "lid_level": tilt <= t.box_tilt_tolerance_rad, + "box_upright": rb[2, 2] >= np.cos(t.box_tilt_tolerance_rad), + "box_not_dropped": s.positions[box][2] >= self.initial.positions[box][2] - 0.03, + "lid_box_contact": s.touching(lid, box), + "released": not s.hand_contact(lid), + "linear_stable": speed <= t.box_max_linear_speed_m_s, + "angular_stable": angular_speed <= t.box_max_angular_speed_rad_s, + } + self.metrics = { + "relative_height_m": relative[2], + "xy_error_m": np.linalg.norm(relative[:2]), + "tilt_rad": tilt, + "linear_speed_m_s": speed, + "angular_speed_rad_s": angular_speed, + } + + def finalize(self): + hold = 0.0 if self.stable_since is None else self.previous.time - self.stable_since + passed = bool(self.checks) and all(self.checks.values()) and hold + 1e-9 >= self.thresholds.hold_s + failed = [k for k, v in self.checks.items() if not v] + if hold + 1e-9 < self.thresholds.hold_s: + failed.append("terminal_hold") + return { + "task": self.task, + "success": passed, + "failed_checks": failed, + "checks": {k: bool(v) for k, v in self.checks.items()}, + "metrics": {k: float(v) for k, v in self.metrics.items()}, + "stable_seconds": float(hold), + "thresholds": asdict(self.thresholds), + "calibration": "provisional", + } diff --git a/benchmarks/sharpa_webxr/API.md b/benchmarks/sharpa_webxr/API.md new file mode 100644 index 00000000..4ce45434 --- /dev/null +++ b/benchmarks/sharpa_webxr/API.md @@ -0,0 +1,156 @@ +# Policy server API + +The eval side (sim env, `rollout.py`, judge) knows nothing about any model. A model plugs in as a **policy +server**: a process that answers two endpoints over ZeroMQ. Everything model-specific lives in the server: +- image layout and resizing; +- state encoding; +- normalization stats; +- the action codec; +- the sampler; +- deploy settings (how many actions to execute, which past observations are needed). + +The server belongs with the model's training code, not in the eval repo. + +Reference implementation: `policies/hold_server.py` (holds the current pose; no GPU). +Base class and transport: `policies/common.py` (`BasePolicy`, `serve`). +Conformance check: `check_server.py --port N`. A server must pass it before it is evaluated. + +## Endpoints + +### `meta` (no input) + +Returns a dict describing how the policy is deployed. The runner reads it once. + +| key | type | meaning | +|---|---|---| +| `model` | str | name, logged with every rollout | +| `codec` | str | the model's action encoding (informational; decoding happens in the server) | +| `chunk` | int | H, actions returned per `inference` call | +| `execute_steps` | int | actions the runner executes before the next call, 1..H. Match the model's upstream deploy. `rollout.py --execute-steps` overrides it. | +| `obs_offsets` | list[int] | observation rows the policy needs, relative to now, all <= 0 and including 0. Example: `[-5, 0]` = now and 5 control steps ago. Rows before the episode start repeat the first observation. | +| `control_hz` | float | control rate the actions are meant for (the sim runs 20 Hz) | +| `upstream` | str | which upstream deploy path the settings copy (free text, logged) | +| `sampler` | dict | sampler settings, logged (e.g. `{"name": "unipc", "steps": 30, "cfg": 1.0}`) | +| `ckpt` | str | checkpoint path, logged | +| `cameras` | dict, optional | `{camera_name: [H, W]}` to render. Default: `ego_view` and `chest_view` at 480x640. | + +### `inference` + +Request: + +``` +{ + "obs": {offset: {"images": {"ego_view": uint8[480,640,3], "chest_view": uint8[480,640,3]}, + "state": canonical (1 row)} # one entry per meta obs_offsets + }, + "task": str, # natural-language instruction + "seed": int, # per-request seed; use it for sampling noise so rollouts are reproducible + "reset": bool, # True on the first request of an episode: clear any per-episode state + "prev_cmd": canonical (1 row) | None, # last pose the runner commanded (None on reset) + "executed": int | None # actions executed since the previous request (None on reset) +} +``` + +Response: + +``` +{ + "pos": float[H, 2, 3], # wrist positions (m) + "rot": float[H, 2, 3, 3], # wrist rotation matrices + "hand": float[H, 2, 22], # hand joint targets (rad) + "frame": "ego_camera", # must equal the observation's frame + "raw": float32[H, width], # model output before decoding, logged for debugging + "timing": {...}, # optional, e.g. {"server_ms": 812.0} + "ckpt": str +} +``` + +`BasePolicy.inference` builds this from `act()`. A subclass only implements +`act(obs, task, seed, prev_cmd) -> (Canonical [H], raw [H, width])`. + +## Canonical pose + +Both state and actions use one canonical layout (`scripts/sharpa_eef/canonical.py`). Arm axis 2 is +`(left, right)`. + +- `pos [T, 2, 3]` wrist position and `rot [T, 2, 3, 3]` wrist orientation, both in the **ego-camera frame**; +- `hand [T, 2, 22]` absolute hand joint angles. + +Actions are **absolute targets in that frame**, not deltas. A model trained on deltas or relative poses decodes +them in its server, using `obs[0]["state"]` (the measured pose now) or `prev_cmd` (the last commanded pose) as +the reference. `check_server.py` flags a first action more than 0.30 m from the current wrist pose; that +usually means a delta was returned. + +## Control loop (what the runner does) + +``` +reset env (scene, seed) +loop until the episode limit (min(1.5 x p90 demo length, 20 s) at 20 Hz): + request = observations at rows now + obs_offsets, task, seed, reset, prev_cmd, executed + response = server.inference(request) # H actions + execute the first execute_steps actions: per step, IK to arm joints (seeded from the previous solve), + hold for 50 ms (3 physics ticks) + prev_cmd = last executed action; executed = execute_steps +``` + +There is no action ensembling or queue blending: each new chunk replaces the previous one. The runner saves the +full qpos trajectory. Judge frames, signals and videos are rendered offline from it (`postprocess.py`, +`render_video.py`), so they do not depend on the server. + +## Writing a server for a new model + +1. Copy `policies/hold_server.py`. Set the meta fields to the model's upstream deploy settings. +2. In `act`: build the model input from `obs` (images per offset, state), normalize with **the training run's + own stats** (load them from the run directory, never from a default path), sample, denormalize, decode to + absolute canonical poses. +3. Start it: `python _server.py --ckpt --port 11600`. +4. Check it: `.venv_webxr/bin/python benchmarks/sharpa_webxr/check_server.py --port 11600`. +5. Evaluate: `RUN= benchmarks/sharpa_webxr/run_suite.sh 11600 0 4 `, then judge with + `postprocess.py --frames-only` + `score.py`. `watch_run.sh` automates this per checkpoint. + +## Eval watcher (`watch_ckpts.sh`) + +`watch_ckpts.sh` evaluates every checkpoint of a run as it lands. It needs only two model-side inputs: the server +script and a command that lists checkpoints. + +``` +benchmarks/sharpa_webxr/watch_ckpts.sh --run \ + --server \ + --list-ckpts "" \ + --done-file /TRAINING_DONE # or --once to evaluate what exists and stop + [--seeds 0-4] [--final-seeds 5-9] [--stride 10000] [--only-steps 50000,100000] [--port 11600] +``` + +The list command prints one line per **complete** checkpoint, in any order. Repeated steps keep their first +line: + +``` + [extra server args...] +``` + +Deciding "complete" is the model side's job, because only the training code knows its save order. Use a file the +trainer writes last, or, failing that, an mtime settle. Examples from the Sharpa b64 runs (these live with each +model's training code, not here): + +``` +# trainer writes meta.json last in each step dir +for d in $OUT/checkpoints/step_*; do [ -f $d/meta.json ] && echo "${d##*step_} $d --stats $OUT/norm_stats.json"; done + +# single-file EMA checkpoints saved with .tmp + os.replace (atomic): existence is enough +for f in $OUT/ema_iter*.pt; do n=$(basename $f .pt); echo "$((10#${n#ema_iter})) $f --weights ema"; done + +# multi-phase run: global step = phase offset + phase step +for d in $OUT/p2/run/step_*; do echo "$((12000 + 10#${d##*step_})) $d/model.safetensors --config $OUT/p2.yaml"; done +``` + +Pass the run's own stats/config through the extra args, so the server never falls back to a default. + +Behaviour: +- **Order.** Oldest step first, one server at a time. Each checkpoint gets `--seeds`; the last one also gets + `--final-seeds` after `--done-file` appears. +- **Retries.** A failed suite is retried up to `MAX_ATTEMPTS` (default 2), then marked `FAILED_`. +- **Background work.** A renderer writes videos for finished rollouts (`VIDEO=1`), and a judge rates each finished + tag (`JUDGE=1`). +- **Restarts.** Everything is resumable from on-disk markers. For a capped container (e.g. Modal's 24 h), give + `--stop-file`: the watcher exits 3 between evaluations once the file exists, and the supervisor starts it again + with the same command. `--sync-cmd` runs before each scan (e.g. to reload a network volume). diff --git a/benchmarks/sharpa_webxr/README.md b/benchmarks/sharpa_webxr/README.md new file mode 100644 index 00000000..ec3340e6 --- /dev/null +++ b/benchmarks/sharpa_webxr/README.md @@ -0,0 +1,154 @@ +# Closed-loop WebXR sim eval for GWP-0.5 Sharpa checkpoints + +This harness runs the GWP-0.5 Sharpa EEF policy (`run_giga_sharpa_train_job.py --action-repr eef`) closed loop in the +WebXR-Teleop MuJoCo sim. It uses the training scenes from the live WebXR store and seeded resets, so every checkpoint +sees the same initial states. Each rollout gets automatic stage signals, 16 keyframes for a VLM judge, and a video. + +## Layout: env, policy server, runner + +- **`env.py` (model-agnostic sim):** `SharpaWebXREnv(scene, obs=ObsConfig(...), action_space=..., eef_frame=...)`. + - `reset(seed) -> obs` and `step(action, observe=True|False|keys) -> (obs, info)`. One action is one 50 ms control + step (20 Hz). + - Action spaces: `joint` (56 joint targets) or `eef` (wrist pose per arm plus hand joints). For `eef`, the env runs + IK, each solve seeded from the previous one, and returns the residuals in `info`. + - Observations are configurable: cameras by name and resolution, plus state keys (`joints`, `eef`, `object_poses`, + `time`). Images render only when a step asks to observe. + - Physics: TacoSim with the teleop app defaults (2400 Hz physics, 60 Hz tick, each target held for 3 ticks). + - Reset: `teleop_app.prepare_scene` with a seeded RNG (arm jitter 0.08, object xy ±5 cm, yaw ±180°, 1 s settle). +- **`policies/_server.py` (one per model, in the training `.venv` on a GPU)** follows the + `policies/common.py` contract. The server owns everything model-specific: image layout, state encoding, + normalization, sampler and action decode. It returns absolute wrist poses plus hand joints in the ego-camera frame. + The full request/response spec is in [API.md](API.md). `policies/hold_server.py` is a no-model reference server + and template, and `check_server.py --port N` checks any server against the contract. +- **`watch_ckpts.sh` (generic eval watcher):** evaluates every checkpoint of any model's run as it lands, given the + model's server script and a checkpoint-listing command (API.md, "Eval watcher"). `watch_run.sh` is the original + GWP-specific watcher (EMA + raw pairs, best_val). + Its `meta` endpoint publishes how the model is deployed upstream: + - `execute_steps`: actions to run before replanning; + - `obs_offsets`: past rows the model needs, e.g. LDA-1B's t−5; + - optionally `cameras`. + To add a model, write a `BasePolicy` subclass with `act()` and `serve()` it. +- **`rollout.py` (one runner for every model):** reads the server's `meta`, observes only at the rows the next request + needs, executes the first `execute_steps` of each chunk, then re-observes; there is no ensembling. + `--execute-steps` overrides the server's value. Episode length is `limits.limit(scene)` = min(`ceil(1.5 × p90)`, + 20 s); signals, judge frames and videos use the same span. The policy only reacts to what it has seen so far, so + longer recordings are truncated offline: their first N steps are exactly what a run capped at N would have produced. + +`policies/gwp_server.py` (GWP-0.5): + +- **Sampling:** 10 flow-matching steps, shift 5, no CFG. `ego_view` and `chest_view` are composed into the training + T-layout at 320x384. The state and actions use the giga codec in the ego-camera frame, and the noise is seeded + per (scene, seed, chunk). +- **Execution:** `execute_steps` = 20 of 32, the same 62.5% / 1.0 s as upstream's 30 of 48 at 30 Hz + (`run_inference_openloop.sh` REPLAN_STEPS). +- **Parity:** the runner reproduces both earlier paths bit for bit. qpos and IK results were identical against the + old `policy_server.py` client-decode path at `--execute-steps 32`, and against `rollout_v2.py` at 20. + +Stage signals, computed in `postprocess.py`: + +- **reach:** a hand comes within 3 cm of an object. +- **grasp:** at least 0.5 s of hand contact while the object is lifted at least 2 cm or carried at least 3 cm. +- **move:** the object is displaced at least 5 cm, or a joint changes by at least 20° or 3 cm. +- **fallen:** an object fell off the table. +- **Rule verifier** (`benchmarks/sharpa/verifiers.py`): runs for the laptop and box-lid scenes. + +Task completion and placement are scored by `judge.py` (Claude, from the keyframes). + +Validated before use: + +- Rendering matches the recorded dataset frames: PSNR 37.7 dB on the ego view and 42.1 dB on the chest view. +- Replaying recorded setpoints reproduces the recorded outcome to within 1.2 mm. +- Policy inputs are bit-identical to the training dataset (`check_policy_parity.py`). + +## Setup + +```bash +# headless rendering (no GPU GL needed) +apt-get install -y libosmesa6 libegl1 libglvnd0 +git submodule update --init third_party/WebXR-Teleop third_party/giga-world-policy + +# sim venv (python 3.12), separate from the training .venv +uv venv .venv_webxr --python 3.12 +uv pip install --python .venv_webxr/bin/python -r third_party/WebXR-Teleop/requirements.txt \ + torch --index-url https://download.pytorch.org/whl/cpu +uv pip install --python .venv_webxr/bin/python pyzmq imageio imageio-ffmpeg av anthropic openai + +# scenes from the live store (needs gsutil access) -> data/webxr_scenes// +.venv_webxr/bin/python benchmarks/sharpa_webxr/scenes.py $(.venv_webxr/bin/python -c \ + "import json;print(' '.join(json.load(open('benchmarks/sharpa_webxr/tasks.json'))))") +``` + +The policy server runs in the training `.venv`. It needs the Wan2.2 VAE at `assets/video_backbone_ckpt/Wan2.2-VAE-Diffusers/vae`. + +## Automatic eval of a training run + +```bash +benchmarks/sharpa_webxr/watch_run.sh [seeds_intermediate=0-4] [seeds_final=5-9] \ + > .sharpa_sim_eval/logs/watch_.log 2>&1 & +``` + +`` is the folder under `.gwp_runs/`. The watcher works like this: + +- It copies each `checkpoint_stepN/transformer_bf16.pt` before training's rotation deletes it. +- It evaluates the EMA (`ema_stepN/`) and the raw weights of every checkpoint, oldest first. Each is served by + `$SERVER` (default `policies/gwp_server.py`) on its own GPU, using about 15 GB next to training. +- It renders videos from one shared low-priority queue, seed 0 first. +- It rates every finished EMA tag with the VLM judge (`score.py`, rubric r2) and writes + `/scores__r2.json` and `.tsv`. +- After training exits, it adds the final seeds for the last step and evaluates the final `best_val`. + +The watcher is resumable: rerunning the same command skips finished work. It prints `WATCH_ALL_DONE` when done. + +| env | default | meaning | +|---|---|---| +| `VIDEO` | `1` | `0` skips `video.mp4` rendering. Signals and keyframes are always written. | +| `VIDEO_SEEDS` | `all` | Comma list of seeds to render, e.g. `0`. | +| `VIDEO_P` | `32` | Parallel renderers. Each holds about 2.3 GB RAM and renders on the CPU with OSMesa at nice 19. | +| `PORT_BASE` | `11600` | EMA server port; raw uses `PORT_BASE+1`. Change it to run two watchers at once. | +| `SERVER` | `policies/gwp_server.py` | Policy server script; any `policies/common.py` server works. | +| `EXECUTE_STEPS` | server meta | Override actions per replan, e.g. `32` for the full chunk. | +| `JUDGE` | `1` | `0` skips the judge. It is also skipped, with a log line, when no OpenAI key is set. | +| `JUDGE_WEIGHTS` | `ema` | Tags to rate: `ema`, `bf16` or `all`. | +| `JUDGE_MODEL` | `gpt-6-luna` | Judge model; `JUDGE_WORKERS` (default 8) sets parallel requests. | +| `PREP_DIR` | `prep_eef_full_v1` | Training prep dir with norm stats and prompts. Must match the run's `--prep-dir`. | + +Assumptions: + +- The run uses the EEF action representation (62-D), 2 views in a T-layout, and horizon 32. Check a new run with + `check_policy_parity.py` first: expect 24/24 identical inputs and a wrist error in the tens of mm or less. +- Training is detected by its `--out-dir .gwp_runs/` argument. +- Rendering is slower than evaluation (about 3 videos/min on a loaded host), so videos lag behind the numbers. + +## Manual pieces + +```bash +# policy server (training venv) +CUDA_VISIBLE_DEVICES=0 .venv/bin/python -u benchmarks/sharpa_webxr/policies/gwp_server.py --ckpt --port 11500 +# one checkpoint: 58 scenes x seeds 0-9, 10 workers +RUN= benchmarks/sharpa_webxr/run_suite.sh 11500 0 9 10 +# one scene (server's execute_steps; --execute-steps 32 executes the full chunk) +.venv_webxr/bin/python benchmarks/sharpa_webxr/rollout.py --port 11500 --scene --seeds 0-2 --out +.venv_webxr/bin/python benchmarks/sharpa_webxr/postprocess.py //seed_0 +# checkpoint parity on val windows +CUDA_VISIBLE_DEVICES=0 .venv/bin/python benchmarks/sharpa_webxr/check_policy_parity.py --ckpt --n 24 --out g3.json +# score a folder with a VLM judge -> /scores__r2.json + .tsv (per task, per seed) +# rubric r2 needs full-res judge frames: 24 times x (head, chest) at 480x640 +.venv_webxr/bin/python benchmarks/sharpa_webxr/postprocess.py --frames-only .sharpa_sim_eval///*/seed_* +.venv_webxr/bin/python benchmarks/sharpa_webxr/score.py .sharpa_sim_eval// --model gpt-6-luna # OPENAI_API_KEY or ~/.config/openai/key +.venv_webxr/bin/python benchmarks/sharpa_webxr/judge.py .sharpa_sim_eval///*/seed_* # Claude judge, ANTHROPIC_API_KEY +# check a policy server against the API before evaluating it (reference: policies/hold_server.py) +.venv_webxr/bin/python benchmarks/sharpa_webxr/check_server.py --port 11600 +# per-category / per-task success (k/n) across checkpoints or models +.venv_webxr/bin/python benchmarks/sharpa_webxr/task_breakdown.py xwam=/scores_gpt-6-luna_r2.json motus=... --tasks +``` + +## Results + +- Table: `.venv/bin/python benchmarks/sharpa_webxr/aggregate.py `. It prints stage rates, the all-three rate with a + 95% Wilson CI, and the gpt_fleet_depth vs simteleop-only split. +- Notebook: `notebooks/_build_gwp_sim_eval_nb.py`, which builds `notebooks/gwp_sim_eval.ipynb`. It covers stage rates, + rate vs training step, outcome breakdown, a per-scene heatmap, IK/fallen/episode length, keyframes and embedded videos. +- Video page: `build_review_page.py --tags --out `. It writes one section per scene with + checkpoints side by side. +- Per rollout: `.sharpa_sim_eval////seed_k/`, containing `rollout.json`, `trajectory.npz`, + `signals.json`, `keyframes/` and `video.mp4`. diff --git a/benchmarks/sharpa_webxr/_build_task_specs.py b/benchmarks/sharpa_webxr/_build_task_specs.py new file mode 100644 index 00000000..aa46642b --- /dev/null +++ b/benchmarks/sharpa_webxr/_build_task_specs.py @@ -0,0 +1,128 @@ +"""Builds task_specs.json: one ordered stage chain per benchmark scene for verify.py. + +Stages are sim checks (see verify.CHECKS) or {"check": "judge"} when only a vision judge can decide the last step +(no liquid / no pudding / no button in the sim). Object names come from the scene layouts; thresholds are first +guesses to be calibrated on data/webxr_episodes (calibrate.py). Keyed by 5-digit scene prefix here, expanded to +the full scene id from tasks.json on write. + +run: python benchmarks/sharpa_webxr/_build_task_specs.py +""" +import json +from pathlib import Path + +HERE = Path(__file__).resolve().parent +TASKS = json.loads((HERE / "tasks.json").read_text()) +CATS = json.loads((HERE / "task_categories.json").read_text()) + +PAN_DISH = [[-0.09, -0.109, -0.016], [0.09, 0.07, 0.022]] # world_pan_01 without its +y handle + + +def reach(o): + return {"name": "reach", "check": "reach", "obj": o} + + +def grasp(o, **kw): + return {"name": "grasp", "check": "grasp", "obj": o, **kw} + + +def pick(o, lift=0.05, hold=1.0, final=False): + # "pick up X" is labelled on ever having held it; "grasp/grip X firmly" (final=True) also requires it was not + # dropped afterwards (still held at the end, or set down gently) + return {"name": "lift_hold", "check": "hold_lifted", "obj": o, "lift_m": lift, "hold_s": hold, **({"end_rule": "not_dropped"} if final else {})} + + +def place(o, t, **kw): + return {"name": "place", "check": "place_in", "obj": o, "target": t, **kw} + + +def judge(what): + return {"name": "complete", "check": "judge", "question": what} + + +S = { + # ---- pick and place ------------------------------------------------------------------------------------ + "00001": ["world_butter", "world_pan_01", [reach("world_butter"), grasp("world_butter"), place("world_butter", "world_pan_01", target_box=PAN_DISH, contact="world_pan_01")]], + "00019": ["world_bread_02", "world_bread_03", [reach("world_bread_02"), grasp("world_bread_02"), place("world_bread_02", "world_bread_03", contact="world_bread_03", xy_margin_m=-0.01)]], + "00286": ["world_bread_03", "world_plate", [reach("world_bread_03"), grasp("world_bread_03"), place("world_bread_03", "world_plate", contact="world_plate")]], + "00381": ["world_flower_01", None, [reach("world_flower_01"), grasp("world_flower_01"), {"name": "out_of_box", "check": "move_out", "obj": "world_flower_01", "min_disp_m": 0.12, "release": False, "rest": False, "final": True}]], + "00391": ["world_plate_04", None, [reach("world_plate_04"), grasp("world_plate_04"), {"name": "out_of_oven", "check": "move_out", "obj": "world_plate_04", "min_disp_m": 0.15, "release": False, "rest": False, "final": True}]], + "00457": ["world_test_tube_01", "world_rack", [reach("world_test_tube_01"), grasp("world_test_tube_01"), place("world_test_tube_01", "world_rack", z_above_m=0.12, contact="world_rack")]], + "00636": ["world_teapot_lid", None, [reach("world_teapot_lid"), grasp("world_teapot_lid"), {"name": "lid_off_and_down", "check": "move_out", "obj": "world_teapot_lid", "min_disp_m": 0.10}]], + "00646": ["world_phone_01", "world_phone_case", [reach("world_phone_01"), grasp("world_phone_01"), {"name": "out_of_case", "check": "move_out", "obj": "world_phone_01", "target": "world_phone_case", "min_disp_m": 0.08, "release": False, "rest": False, "final": True}]], + "00807": ["world_egg", "world_pan_01", [reach("world_egg"), grasp("world_egg"), place("world_egg", "world_pan_01", target_box=PAN_DISH, contact="world_pan_01")]], + "00921": ["world_butter", "world_pan", [reach("world_butter"), grasp("world_butter"), place("world_butter", "world_pan", xy_margin_m=0.03, contact="world_pan")]], + "01012": ["world_eyeshadow_brush", "world_eyeshadow_box", [reach("world_eyeshadow_brush"), grasp("world_eyeshadow_brush"), place("world_eyeshadow_brush", "world_eyeshadow_box", z_above_m=0.04, contact="world_eyeshadow_box")]], + "01083": ["world_coin", "world_game_box", [reach("world_coin"), grasp("world_coin"), place("world_coin", "world_game_box", z_above_m=0.03, contact="world_game_box")]], + "01085": ["world_coin", "world_game_box", [reach("world_coin"), grasp("world_coin"), place("world_coin", "world_game_box", z_above_m=0.03, contact="world_game_box")]], + "01089": ["world_dice", "world_game_board", [reach("world_dice"), grasp("world_dice"), place("world_dice", "world_game_board", contact="world_game_board")]], + "01242": ["world_bowl", None, [judge("banana pieces transferred into the salad bowl (no banana bodies in the sim)")]], + "01440": ["world_shovel", "world_pot", [reach("world_shovel"), grasp("world_shovel"), place("world_shovel", "world_pot", z_above_m=0.12, contact="world_pot")]], + "self-": ["synthesis_cup_cup_005_5e43862263a34727", None, [reach("synthesis_cup_cup_005_5e43862263a34727"), grasp("synthesis_cup_cup_005_5e43862263a34727"), judge("cup placed on the cup rack (rack not simulated)")]], + # ---- articulated open/close ---------------------------------------------------------------------------- + "00005": ["world_laptop_03", None, [reach("world_laptop_03"), {"name": "adjusted", "check": "joint_goal", "obj": "world_laptop_03", "delta": -0.15, "min": -0.6, "max": 0.12, "release": True, "hold_s": 1.0, "final": True}]], + "00079": ["world_gate_01", None, [reach("world_gate_01"), {"name": "closed", "check": "joint_goal", "obj": "world_gate_01", "joint": "articraft_rec_microwave_oven_0002_joints_root_to_door", "max": 0.06, "hold_s": 0.5, "final": True}]], + "00082": ["world_laptop_03", None, [reach("world_laptop_03"), {"name": "closed", "check": "joint_goal", "obj": "world_laptop_03", "min": 1.85, "hold_s": 0.5, "max_speed": 4.6, "final": True}]], + "00337": ["world_laptop_03", None, [reach("world_laptop_03"), {"name": "opened", "check": "joint_goal", "obj": "world_laptop_03", "max": 1.25, "hold_s": 1.0, "final": True}]], + "00626": ["world_rec_desk_with_drawer_0005", None, [reach("world_rec_desk_with_drawer_0005"), {"name": "pulled_out", "check": "joint_goal", "obj": "world_rec_desk_with_drawer_0005", "min": 0.08, "hold_s": 0.5, "final": True}]], + "01000": ["world_burger_box", None, [reach("world_burger_box"), {"name": "opened", "check": "joint_goal", "obj": "world_burger_box", "delta": -0.9, "hold_s": 0.5, "final": True}]], + "01018": ["world_box", None, [reach("world_box"), {"name": "closed", "check": "joint_goal", "obj": "world_box", "max": 0.15, "hold_s": 0.3}, + {"name": "faces_robot", "check": "orientation", "obj": "world_box", "mode": "face", "axis": [1, 0, 0], "toward": [-0.035, 0.999], "tol_deg": 15, "keep_upright_deg": 30, "hold_s": 0.5, "final": True}]], + "01747": ["world_pencil_case", None, [reach("world_pencil_case"), {"name": "opened", "check": "joint_goal", "obj": "world_pencil_case", "max": -1.0, "hold_s": 0.5, "final": True}]], + # ---- cap / lid --------------------------------------------------------------------------------------- + "00023": ["world_lid_09", "world_box", [reach("world_lid_09"), grasp("world_lid_09"), {"name": "lid_on", "check": "relative_pose", "obj": "world_lid_09", "ref": "world_box", "point": [0, 0, 0.004], "xy_tol_m": 0.03, "z_tol_m": 0.02, "z_above_m": 0.03, "tilt_deg": 12, "contact": "world_box"}]], + "00075": ["world_pen_cap", "world_pen", [reach("world_pen_cap"), grasp("world_pen_cap"), {"name": "capped", "check": "relative_pose", "obj": "world_pen_cap", "ref": "world_pen", "point": [0.038, 0, 0.016], "offset": [0.041, 0.002, 0.012], "pos_tol_m": 0.012, "tilt_deg": 90, "contact": "world_pen", "release": False, "rest": False}]], + "00077": ["world_teapot_lid", "world_teapot", [reach("world_teapot_lid"), grasp("world_teapot_lid"), {"name": "lid_on", "check": "relative_pose", "obj": "world_teapot_lid", "ref": "world_teapot", "xy_tol_m": 0.035, "z_tol_m": 0.04, "z_above_m": 0.05, "tilt_deg": 15, "contact": "world_teapot"}]], + "00894": ["world_cap_07", "world_body_03", [reach("world_cap_07"), grasp("world_cap_07"), {"name": "capped", "check": "relative_pose", "obj": "world_cap_07", "ref": "world_body_03", "point": [0, 0, 0.083], "xy_tol_m": 0.03, "z_tol_m": 0.03, "z_above_m": 0.04, "tilt_deg": 15, "contact": "world_body_03"}]], + # ---- pick / grasp only -------------------------------------------------------------------------------- + "00177": ["world_ukelele_01", None, [reach("world_ukelele_01"), pick("world_ukelele_01", final=True)]], + "00178": ["world_sauce_01", None, [reach("world_sauce_01"), pick("world_sauce_01", final=True)]], + "00375": ["world_bag_02", None, [reach("world_bag_02"), pick("world_bag_02")]], + "01116": ["world_drill", None, [reach("world_drill"), pick("world_drill", final=True)]], + "01174": ["world_stamp", None, [reach("world_stamp"), pick("world_stamp")]], + "01304": ["world_kettle", None, [reach("world_kettle"), pick("world_kettle")]], + "01337": ["world_scissors", None, [reach("world_scissors"), pick("world_scissors")]], + # ---- reorient / flip / rotate ------------------------------------------------------------------------- + "01032": ["world_box", None, [reach("world_box"), {"name": "upturned", "check": "orientation", "obj": "world_box", "mode": "flip", "tol_deg": 35, "final": True, "release": False, "rest": False}]], + "01041": ["world_game_board", None, [reach("world_game_board"), {"name": "flipped", "check": "orientation", "obj": "world_game_board", "mode": "flip", "tol_deg": 35, "final": True, "release": False, "rest": False}]], + "01246": ["world_bowl", None, [reach("world_bowl"), {"name": "upturned", "check": "orientation", "obj": "world_bowl", "mode": "flip", "tol_deg": 45, "final": True, "release": False, "rest": False}]], + "01541": ["world_box", None, [reach("world_box"), grasp("world_box", lift_m=0.02, carry_m=0.03), {"name": "rotated_side", "check": "orientation", "obj": "world_box", "mode": "rotate", "min_deg": 60}]], + "01542": ["world_teacup", None, [reach("world_teacup"), {"name": "handle_faces_robot", "check": "orientation", "obj": "world_teacup", "mode": "face", "axis": [1, 0, 0], "toward": [1, 0], "tol_deg": 25, "keep_upright_deg": 30, "final": True}]], + "01564": ["world_tablet", None, [reach("world_tablet"), {"name": "portrait", "check": "orientation", "obj": "world_tablet", "mode": "rotate", "min_deg": 55, "final": True, "release": False, "rest": False}]], + # ---- shake / toss ------------------------------------------------------------------------------------- + "00779": ["world_flask_02", None, [reach("world_flask_02"), grasp("world_flask_02"), {"name": "shaken", "check": "oscillation", "obj": "world_flask_02", "min_reversals": 4, "min_amp_m": 0.03}]], + "01592": ["world_dice", None, [reach("world_dice"), grasp("world_dice", lift_m=0.02), {"name": "shaken", "check": "oscillation", "obj": "world_dice", "min_reversals": 4, "min_amp_m": 0.02}]], + "01605": ["world_nail_polish_bottle", None, [reach("world_nail_polish_bottle"), grasp("world_nail_polish_bottle"), {"name": "shaken", "check": "oscillation", "obj": "world_nail_polish_bottle", "min_reversals": 4, "min_amp_m": 0.03}]], + "01652": ["world_coin", None, [reach("world_coin"), grasp("world_coin", lift_m=0.02), {"name": "tossed_caught", "check": "free_flight", "obj": "world_coin", "min_rise_m": 0.04, "min_rel_rise_m": 0.02, "catch_dist_m": 0.22}]], + # ---- pour (sim: grasp, carry over, tilt; judge: the pour itself) ---------------------------------------- + "00175": ["world_kettle", "world_mug_025", [reach("world_kettle"), grasp("world_kettle"), {"name": "tilt_over_target", "check": "tilt_over", "obj": "world_kettle", "target": "world_mug_025", "min_tilt_deg": 30, "xy_slack_m": 0.12}, judge("water poured from the kettle into the mug")]], + "00475": ["world_beaker_01", "world_flask_02", [reach("world_beaker_01"), grasp("world_beaker_01"), {"name": "tilt_over_target", "check": "tilt_over", "obj": "world_beaker_01", "target": "world_flask_02", "min_tilt_deg": 50, "xy_slack_m": 0.08}, judge("substance poured from the beaker into the conical flask")]], + "00487": ["world_body_04", "world_bowl", [reach("world_body_04"), grasp("world_body_04"), {"name": "tilt_over_target", "check": "tilt_over", "obj": "world_body_04", "target": "world_bowl", "min_tilt_deg": 50, "xy_slack_m": 0.08}, judge("tea poured from the bottle into the bowl")]], + "01480": ["world_bowl", "world_pan", [reach("world_bowl"), grasp("world_bowl"), {"name": "tilt_over_target", "check": "tilt_over", "obj": "world_bowl", "target": "world_pan", "min_tilt_deg": 50, "xy_slack_m": 0.05}, judge("bowl contents poured into the pan")]], + "01498": ["world_kettle", "world_teapot", [reach("world_kettle"), grasp("world_kettle"), {"name": "tilt_over_target", "check": "tilt_over", "obj": "world_kettle", "target": "world_teapot", "min_tilt_deg": 30, "xy_slack_m": 0.12}, judge("hot water poured from the kettle into the teapot")]], + # ---- tool use (sim: grasp tool, tool acts on target; judge: the effect) --------------------------------- + "00051": ["world_eraser_07", "world_plate_05", [reach("world_eraser_07"), grasp("world_eraser_07", lift_m=0.01, carry_m=0.03), {"name": "brushed", "check": "sweep", "obj": "world_eraser_07", "target": "world_plate_05", "contact": True, "min_path_m": 0.15}, judge("plate brushed with the eraser")]], + "00749": ["world_spoon_08", "world_pan_06", [reach("world_spoon_08"), grasp("world_spoon_08"), {"name": "stirred", "check": "sweep", "obj": "world_spoon_08", "target": "world_pan_06", "xy_slack_m": 0.03, "z_above_m": 0.12, "min_path_m": 0.30}, judge("pot stirred with the spoon")]], + "00787": ["world_spatula", "world_pan_01", [reach("world_spatula"), grasp("world_spatula"), {"name": "scrambled", "check": "sweep", "obj": "world_spatula", "target": "world_pan_01", "target_box": [[-0.076, -0.097, -0.019], [0.072, 0.09, 0.042]], "xy_slack_m": 0.03, "z_above_m": 0.10, "min_path_m": 0.30}, judge("egg scrambled by moving the spatula around the pan")]], + "00863": ["world_hammer_03", "world_box_13", [reach("world_hammer_03"), grasp("world_hammer_03"), {"name": "hit", "check": "contact_events", "obj": "world_hammer_03", "target": "world_box_13", "min_events": 1, "min_speed_m_s": 0.15}, judge("box hit with the hammer head")]], + "01477": ["world_camera", None, [reach("world_camera"), pick("world_camera", lift=0.03, hold=0.5, final=False), judge("camera button clicked (no button in the sim)")]], + "01587": ["world_spoon", "world_plate", [reach("world_spoon"), grasp("world_spoon"), {"name": "scooped", "check": "sweep", "obj": "world_spoon", "target": "world_plate", "xy_slack_m": 0.05, "z_above_m": 0.08, "min_path_m": 0.05}, judge("a piece of pudding scooped from the plate with the spoon")]], + "01640": ["world_drumstick", "world_drum", [reach("world_drumstick"), grasp("world_drumstick"), {"name": "hit", "check": "contact_events", "obj": "world_drumstick", "target": "world_drum", "min_events": 1, "min_speed_m_s": 0.15}, judge("drumhead hit with the drum stick")]], +} + + +def main(): + full = {sid[:5]: sid for sid in TASKS} + assert set(S) == set(full), (set(S) ^ set(full)) + out = {} + for short, (obj, target, stages) in S.items(): + sid = full[short] + out[sid] = {"version": 1, "category": CATS[sid], "task": TASKS[sid]["task"], "task_object": obj, "target": target, + "no_fall": [o for o in (obj, target) if o], "stages": stages, + "needs_judge": any(s["check"] == "judge" for s in stages)} + (HERE / "task_specs.json").write_text(json.dumps(out, indent=1)) + n_j = sum(v["needs_judge"] for v in out.values()) + print(f"wrote {len(out)} specs, {n_j} end in a judge stage") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/sharpa_webxr/aggregate.py b/benchmarks/sharpa_webxr/aggregate.py new file mode 100644 index 00000000..71ae5767 --- /dev/null +++ b/benchmarks/sharpa_webxr/aggregate.py @@ -0,0 +1,73 @@ +"""Per-checkpoint stage rates for one sim-eval run dir (.sharpa_sim_eval///**/seed_k/signals.json). + +usage: aggregate.py [--tags t1 t2 ...] [--tsv out.tsv] +Prints reach / grasp / move / all-three (with 95% Wilson CI) / fallen / IK, overall and split by scenes that have +gpt_fleet_depth data vs simteleop-only scenes. Searches recursively, so sorted subfolders (e.g. gpt_fleet_depth/) count. +""" +import argparse +import glob +import json +import math +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +TASKS = json.load(open(ROOT / "benchmarks/sharpa_webxr/tasks.json")) +STAGES = ("reach", "grasp", "move") + + +def wilson(k, n, z=1.96): + if n == 0: + return float("nan"), float("nan") + p = k / n + d = 1 + z * z / n + c = (p + z * z / (2 * n)) / d + h = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / d + return c - h, c + h + + +def load(run_dir: Path, tag: str): + return [json.load(open(f)) for f in glob.glob(str(run_dir / tag / "**" / "seed_*" / "signals.json"), recursive=True)] + + +def summarize(sigs): + n = len(sigs) + if n == 0: + return None + full = sum(all(s["stage"].get(k) for k in STAGES) for s in sigs) + lo, hi = wilson(full, n) + out = {"n": n, **{k: sum(bool(s["stage"].get(k)) for s in sigs) / n for k in STAGES}, + "all3": full / n, "all3_lo": lo, "all3_hi": hi, + "fallen": sum(bool(s.get("fallen_any")) for s in sigs) / n, + "ik": sum(s.get("ik_converged_frac", 0) for s in sigs) / n} + for name, want in (("gpt", True), ("teleop", False)): + sub = [s for s in sigs if bool(TASKS[s["scene_id"]]["n_eps"].get("gpt_fleet_depth")) == want] + out[f"all3_{name}"] = sum(all(s["stage"].get(k) for k in STAGES) for s in sub) / max(len(sub), 1) + return out + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("run") + p.add_argument("--tags", nargs="*") + p.add_argument("--tsv", type=Path) + a = p.parse_args() + run_dir = ROOT / ".sharpa_sim_eval" / a.run + tags = a.tags or sorted(d.name for d in run_dir.iterdir() if d.is_dir() and d.name != "g3") + cols = ["n", "reach", "grasp", "move", "all3", "all3_lo", "all3_hi", "all3_gpt", "all3_teleop", "fallen", "ik"] + rows = [] + print(f"{'tag':28s} " + " ".join(f"{c:>8s}" for c in cols)) + for t in tags: + r = summarize(load(run_dir, t)) + if r is None: + continue + rows.append((t, r)) + print(f"{t:28s} " + " ".join(f"{r[c]:8d}" if c == "n" else f"{r[c]:8.3f}" for c in cols)) + if a.tsv: + with open(a.tsv, "w") as f: + f.write("tag\t" + "\t".join(cols) + "\n") + for t, r in rows: + f.write(t + "\t" + "\t".join(str(r[c]) for c in cols) + "\n") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/sharpa_webxr/build_review_page.py b/benchmarks/sharpa_webxr/build_review_page.py new file mode 100644 index 00000000..e6c8307a --- /dev/null +++ b/benchmarks/sharpa_webxr/build_review_page.py @@ -0,0 +1,134 @@ +"""Video review page for sim-eval rollouts: one section per scene, one video card per checkpoint tag, side by side. + +usage: build_review_page.py --tags step005000_ema step005000_bf16 [--seeds 0] --out +Writes /index.html and /v//_s.mp4 (re-encoded small for the web), plus /files.json +(the published paths, for publishing in batches under the 64 MB per-publish limit). Rollouts without a rendered +video.mp4 yet are shown as "video pending" and picked up on the next build. +""" +import argparse +import glob +import html +import json +import subprocess +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +import imageio_ffmpeg + +ROOT = Path(__file__).resolve().parents[2] +HERE = Path(__file__).resolve().parent +TASKS = json.load(open(HERE / "tasks.json")) +STAGES = ("reach", "grasp", "move") + + +def find(run_dir, tag, scene, seed): + hits = glob.glob(str(run_dir / tag / "**" / scene / f"seed_{seed}"), recursive=True) + return Path(hits[0]) if hits else None + + +def full(s): + return all(s["stage"].get(k) for k in STAGES) + + +def encode(src, dst): + if dst.exists() and dst.stat().st_mtime >= src.stat().st_mtime: + return + dst.parent.mkdir(parents=True, exist_ok=True) + subprocess.run([imageio_ffmpeg.get_ffmpeg_exe(), "-y", "-loglevel", "error", "-i", str(src), "-c:v", "libx264", + "-crf", "30", "-preset", "slow", "-pix_fmt", "yuv420p", "-movflags", "+faststart", str(dst)], check=True) + + +def chip(ok, name): + return f'{name} {"yes" if ok else "no"}' + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("run") + p.add_argument("--tags", nargs="+", required=True) + p.add_argument("--seeds", nargs="+", type=int, default=[0]) + p.add_argument("--out", type=Path, required=True) + p.add_argument("--title", default=None) + a = p.parse_args() + run_dir = ROOT / ".sharpa_sim_eval" / a.run + a.out.mkdir(parents=True, exist_ok=True) + + sigs = {t: {} for t in a.tags} # tag -> scene -> [signals] + for t in a.tags: + for f in glob.glob(str(run_dir / t / "**" / "seed_*" / "signals.json"), recursive=True): + s = json.load(open(f)) + sigs[t].setdefault(s["scene_id"], []).append(s) + + jobs, files = [], [] + secs = [] + order = sorted(TASKS, key=lambda sc: -sum(sum(map(full, sigs[t].get(sc, []))) / max(len(sigs[t].get(sc, [])), 1) for t in a.tags)) + for sc in order: + pre = sc.split("_")[0] + cards = [] + for t in a.tags: + for k in a.seeds: + d = find(run_dir, t, sc, k) + s = json.load(open(d / "signals.json")) if d and (d / "signals.json").exists() else None + rate = sigs[t].get(sc, []) + rate_s = f"{sum(map(full, rate))}/{len(rate)} seeds all three" if rate else "not run yet" + if s is None: + cards.append(f'
not run yet
{t} · seed {k}
') + continue + r = json.load(open(d / "rollout.json")) + st = s["stage"] + chips = "".join(chip(st.get(x), x) for x in STAGES) + if s.get("fallen_any"): + chips += 'object fell' + if s.get("verifier"): + chips += chip(s["verifier"].get("success_any"), "verifier") + rel = f"v/{t}/{pre}_s{k}.mp4" + if (d / "video.mp4").exists(): + jobs.append((d / "video.mp4", a.out / rel)) + files.append(rel) + vid = f'' + else: + vid = '
video rendering, check back later
' + cards.append(f'
{vid}
{html.escape(t)} · seed {k}' + f'{r["steps"] / 20:.1f} s · IK {s.get("ik_converged_frac", 0) * 100:.0f}%
' + f'
{chips}
{rate_s}
') + star = "★ " if TASKS[sc]["n_eps"].get("gpt_fleet_depth") else "" + secs.append(f'

{star}{html.escape(TASKS[sc]["task"])}

' + f'{html.escape(sc)}
{"".join(cards)}
') + + with ThreadPoolExecutor(32) as ex: + list(ex.map(lambda j: encode(*j), jobs)) + + rows = [] + for t in a.tags: + ss = [s for v in sigs[t].values() for s in v] + n = len(ss) + if not n: + rows.append(f"{t}0" + "–" * 6 + "") + continue + m = lambda f: sum(map(f, ss)) / n # noqa: E731 + rows.append(f"{t}{n}" + "".join(f"{m(lambda s: bool(s['stage'].get(x))):.2f}" for x in STAGES) + + f"{m(full):.2f}{m(lambda s: bool(s.get('fallen_any'))):.2f}{m(lambda s: s.get('ik_converged_frac', 0)):.2f}") + css = (HERE / "review_page.css").read_text() + title = a.title or f"{a.run} Sim Videos" + page = f"""{html.escape(title)} + + + +
+

{html.escape(title)}

+

Closed-loop WebXR MuJoCo sim, {len(TASKS)} training scenes. Cards in each row compare checkpoints on the same seeded reset. Each video shows the ego view on the left and the chest view on the right. Scenes are sorted best to worst; ★ marks scenes with gpt_fleet_depth data.

+
run {html.escape(a.run)}seeds shown {", ".join(map(str, a.seeds))}chunk 32 steps, executed fullyflow steps 10control 20 Hz
+

Stage rates over all evaluated seeds

+
{"".join(rows)}
checkpointrolloutsreachgraspmoveall threeobject fellIK converged
+

These are sim signals only. Reach means a hand came within 3 cm of an object. Grasp means 0.5 s or more of contact while the object was lifted or carried. Move means an object moved 5 cm or more, or a joint turned 20° or more. Task completion waits on the VLM judge.

+{"".join(secs)} +
""" + (a.out / "index.html").write_text(page) + json.dump(files, open(a.out / "files.json", "w")) + print(f"wrote {a.out / 'index.html'}: {len(files)} videos, {sum((a.out / f).stat().st_size for f in files) / 1e6:.1f} MB") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/sharpa_webxr/calibrate.py b/benchmarks/sharpa_webxr/calibrate.py new file mode 100644 index 00000000..0824c89d --- /dev/null +++ b/benchmarks/sharpa_webxr/calibrate.py @@ -0,0 +1,118 @@ +"""Verifier vs human labels on data/webxr_episodes (success / recover = completed, failure = not completed). + +usage: calibrate.py [--root data/webxr_episodes] [--tsv out.tsv] [--show-errors N] +Per scene: n by label, recall on completed (success_any true), specificity on failures, per-stage achieved rate +split by label, and the worst-offending stage. Scenes whose chain ends in a judge stage are scored on sim_prefix_ok. +""" +from __future__ import annotations + +import argparse +import json +from collections import defaultdict +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +TASKS = json.loads((Path(__file__).resolve().parent / "tasks.json").read_text()) + + +def load(root: Path): + rows = [] + for f in root.glob("*/*/verifier.json"): + v = json.loads(f.read_text()) + if "error" in v: + rows.append({"scene_id": v["scene_id"], "dir": str(f.parent), "error": v["error"]}) + continue + meta = json.loads((f.parent / "rollout.json").read_text()) + v["label"] = meta["outcome"] + v["completed"] = meta["outcome"] != "failure" + v["dir"] = str(f.parent) + rows.append(v) + return rows + + +def rate(xs): + return round(sum(xs) / len(xs), 3) if xs else None + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--root", type=Path, default=ROOT / "data/webxr_episodes") + p.add_argument("--tsv", type=Path) + p.add_argument("--show-errors", type=int, default=0, help="print N false negatives / positives per scene") + p.add_argument("--scenes", nargs="*") + a = p.parse_args() + rows = load(a.root) + by = defaultdict(list) + for r in rows: + if not a.scenes or any(r["scene_id"].startswith(s) for s in a.scenes): + by[r["scene_id"]].append(r) + out = [] + print(f"{'scene':8s} {'category':24s} {'n_s/r/f':>9s} {'recall':>7s} {'spec':>6s} {'rec_s':>6s} {'rec_r':>6s} stage any/final rate: completed|failure") + tot = {"tp": 0, "fn": 0, "tn": 0, "fp": 0} + tot_sim = {"tp": 0, "fn": 0, "tn": 0, "fp": 0} # scenes whose chain is fully sim-verifiable + for sid in sorted(by): + rs = [r for r in by[sid] if "error" not in r] + errs = [r for r in by[sid] if "error" in r] + if not rs: + print(sid[:8], "all errors:", errs[0]["error"][:80] if errs else "") + continue + pos = [r for r in rs if r["completed"]] + neg = [r for r in rs if not r["completed"]] + key = "sim_prefix_ok" + tp = sum(r[key] for r in pos); fn = len(pos) - tp + fp = sum(r[key] for r in neg); tn = len(neg) - fp + for k, v in zip(("tp", "fn", "tn", "fp"), (tp, fn, tn, fp)): + tot[k] += v + if not rs[0]["needs_judge"]: + tot_sim[k] += v + stages = [s["name"] for s in rs[0]["stages"]] + st = [] + for i, name in enumerate(stages): + if rs[0]["stages"][i]["check"] == "judge": + st.append(f"{name}:J") + continue + ap = rate([r["stages"][i]["achieved"] for r in pos]); an = rate([r["stages"][i]["achieved"] for r in neg]) + fp_ = rate([bool(r["stages"][i]["final"]) for r in pos]); fn_ = rate([bool(r["stages"][i]["final"]) for r in neg]) + st.append(f"{name}:{ap}/{fp_}|{an}/{fn_}") + n_s = sum(r["label"] == "success" for r in rs); n_r = sum(r["label"] == "recover" for r in rs) + rec = {"scene": sid[:8], "category": rs[0].get("category"), "n_success": n_s, "n_recover": n_r, "n_failure": len(neg), + "recall": rate([r[key] for r in pos]), "specificity": rate([not r[key] for r in neg]), + "recall_success": rate([r[key] for r in pos if r["label"] == "success"]), + "recall_recover": rate([r[key] for r in pos if r["label"] == "recover"]), + "fallen_rate": rate([any(r["fallen"].values()) for r in rs]), "n_errors": len(errs), "stages": " ".join(st)} + out.append(rec) + print(f"{rec['scene']:8s} {str(rec['category'])[:24]:24s} {n_s:3d}/{n_r:2d}/{len(neg):2d} {str(rec['recall']):>7s} {str(rec['specificity']):>6s} " + f"{str(rec['recall_success']):>6s} {str(rec['recall_recover']):>6s} {rec['stages']}" + (f" ERR={len(errs)}" if errs else "")) + if a.show_errors: + for r in [r for r in pos if not r[key]][: a.show_errors]: + print(" FN", r["dir"].split("/")[-1], [(s["name"], s["achieved"]) for s in r["stages"]], {s["name"]: s["metrics"] for s in r["stages"] if not s["achieved"]}) + for r in [r for r in neg if r[key]][: a.show_errors]: + print(" FP", r["dir"].split("/")[-1], [(s["name"], round(s["t_first_s"], 1) if s["t_first_s"] is not None else None) for s in r["stages"]]) + # same totals under pure any-time and pure end-state semantics (sim-only scenes) + def any_time(r): + return all(s["any_time"] for s in r["stages"]) and not any(r["fallen"].values()) + + def end_state(r): # earlier stages any time, the goal stage still true at the end of the episode + return all(s["any_time"] for s in r["stages"][:-1]) and r["stages"][-1]["end_state"] and not any(r["fallen"].values()) + + for fn_, label in ((any_time, "SIM-ONLY, any-time semantics"), (end_state, "SIM-ONLY, end-state semantics")): + rs = [r for sid in by for r in by[sid] if "error" not in r and not r["needs_judge"] and "any_time" in r["stages"][0]] + if rs: + tp = sum(fn_(r) for r in rs if r["completed"]); fn = sum(not fn_(r) for r in rs if r["completed"]) + fp = sum(fn_(r) for r in rs if not r["completed"]); tn = sum(not fn_(r) for r in rs if not r["completed"]) + print(f"{label}: recall={tp / max(1, tp + fn):.3f} specificity={tn / max(1, tn + fp):.3f} accuracy={(tp + tn) / len(rs):.3f}") + for name, t in (("ALL (judge scenes scored on sim prefix)", tot), ("SIM-ONLY scenes", tot_sim)): + n = sum(t.values()) + if n: + print(f"\n{name}: n={n} recall={t['tp'] / max(1, t['tp'] + t['fn']):.3f} specificity={t['tn'] / max(1, t['tn'] + t['fp']):.3f} " + f"accuracy={(t['tp'] + t['tn']) / n:.3f} (tp={t['tp']} fn={t['fn']} tn={t['tn']} fp={t['fp']})") + if a.tsv: + cols = list(out[0].keys()) + with open(a.tsv, "w") as f: + f.write("\t".join(cols) + "\n") + for r in out: + f.write("\t".join(str(r[c]) for c in cols) + "\n") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/sharpa_webxr/check_policy_parity.py b/benchmarks/sharpa_webxr/check_policy_parity.py new file mode 100644 index 00000000..8a9b5595 --- /dev/null +++ b/benchmarks/sharpa_webxr/check_policy_parity.py @@ -0,0 +1,77 @@ +"""G3: eval policy path == training path on val windows. + +(1) images/state/prompt from GWPSharpaPolicy.preprocess (raw frames) are bit-identical to the training dataset's + __getitem__; (2) training flow-matching action loss of the loaded weights on val windows (compare to the + run's val/action_loss); (3) sampled chunk (eval sampler, 10 and 20 steps) vs ground truth: normalized MSE and + decoded absolute wrist position / rotation / hand-joint errors. +""" +import argparse +import json +import sys + +import numpy as np +import torch + +sys.path.insert(0, "benchmarks/sharpa_webxr") +from policy_server import GWPSharpaPolicy # noqa: E402 + +p = argparse.ArgumentParser() +p.add_argument("--ckpt", required=True) +p.add_argument("--n", type=int, default=24) +p.add_argument("--out", required=True) +a = p.parse_args() + +pol = GWPSharpaPolicy(a.ckpt, num_steps=10) +T = pol.T +T.SPLIT_FILE = "assets/sharpa_full_meta/split_v1.json" +dirs = ["assets/sharpa_full_meta/gpt_fleet_depth", "assets/sharpa_full_meta/simteleop_1004_fleet"] +norm = {"observation.state": {"q01": pol.s_lo, "q99": pol.s_hi}, "action": {"q01": pol.a_lo, "q99": pol.a_hi}} +ds = T.SharpaGigaEEFDataset("val", None, norm, pol.prompt_embeds, False, dirs) +rng = np.random.default_rng(0) +pool = rng.choice(ds.pool_size(), size=a.n, replace=False) +ds.indices = pool + +from giga_train import ModuleDict # noqa: E402 +loss_ns = T.build_loss_ns(ModuleDict({"transformer": pol.model}), pol.vae, pol.device) +res = {"ckpt": a.ckpt, "n": a.n, "identical_inputs": [], "flow_action_loss": [], "s10": [], "s20": []} +A = ds.A +for i, w in enumerate(pool): + item = ds[i] + s = ds.sample(int(w)) + # (1) preprocess parity + imgs = {c.split(".")[-1]: np.asarray(s["video"][c][0]) for c in T.EEF_CAMERAS} + state = ds.codec.state_encode(s["state"])[0] + raw = pol.preprocess(imgs, state, s["prompt"]) + same = (torch.equal(raw["images"][0, 0], item["images"][0]) and torch.allclose(raw["state"][0], item["state"]) + and torch.equal(raw["prompt_embeds"][0], item["prompt_embeds"])) + res["identical_inputs"].append(bool(same)) + # (2) training loss (train mode = training contract), fixed noise + pol.model.train() + with torch.no_grad(), torch.random.fork_rng(devices=[pol.device]): + torch.manual_seed(1234 + i) + batch = T.to_batch({k: v[None] for k, v in item.items()}, pol.device) + res["flow_action_loss"].append(float(loss_ns.forward_step(batch)["action_loss"])) + pol.model.eval() + # (3) sampled chunk vs GT + gt = (item["action"].numpy() + 1) / 2 * np.maximum(pol.a_hi - pol.a_lo, 1e-8) + pol.a_lo + gt_abs = ds.codec.decode(gt, **A.decode_kwargs(s, ds.codec)) + for key, steps in (("s10", 10), ("s20", 20)): + pred = pol.predict(imgs, state, s["prompt"], seed=i, num_steps=steps) + pn = (pred - pol.a_lo) / np.maximum(pol.a_hi - pol.a_lo, 1e-8) * 2 - 1 + pa = ds.codec.decode(pred, **A.decode_kwargs(s, ds.codec)) + dR = np.einsum("tkji,tkjl->tkil", pa.rot, gt_abs.rot) + ang = np.degrees(np.arccos(np.clip((np.trace(dR, axis1=-2, axis2=-1) - 1) / 2, -1, 1))) + res[key].append({"mse_norm": float(((pn - item["action"].numpy()) ** 2).mean()), + "pos_err_mm": float(np.linalg.norm(pa.pos - gt_abs.pos, axis=-1).mean() * 1e3), + "pos_err_mm_last": float(np.linalg.norm(pa.pos[-1] - gt_abs.pos[-1], axis=-1).mean() * 1e3), + "rot_err_deg": float(ang.mean()), + "hand_err_deg": float(np.degrees(np.abs(pa.hand - gt_abs.hand)).mean())}) + print(i, res["identical_inputs"][-1], round(res["flow_action_loss"][-1], 4), res["s10"][-1], flush=True) + +summary = {"identical_inputs": int(sum(res["identical_inputs"])), "n": a.n, + "flow_action_loss_mean": float(np.mean(res["flow_action_loss"]))} +for key in ("s10", "s20"): + summary[key] = {m: float(np.mean([r[m] for r in res[key]])) for m in res[key][0]} +res["summary"] = summary +json.dump(res, open(a.out, "w"), indent=1) +print(json.dumps(summary, indent=1)) diff --git a/benchmarks/sharpa_webxr/check_server.py b/benchmarks/sharpa_webxr/check_server.py new file mode 100644 index 00000000..d9fd35c6 --- /dev/null +++ b/benchmarks/sharpa_webxr/check_server.py @@ -0,0 +1,108 @@ +"""Conformance check for a policy server (policies/common.py contract). Runs in .venv_webxr, like rollout.py. + +Calls "meta" and validates it, then sends real observations from one scene (reset + one follow-up request, at +every row in meta obs_offsets) and validates each response: shapes, finiteness, rotations, frame, distance from +the current pose, latency. A server must pass this before it is evaluated. + +usage: check_server.py --port 11600 [--scene ] [--max-jump 0.30] +""" +from __future__ import annotations + +import argparse +import sys +import time + +import numpy as np + +from rollout import TASKS, PolicyClient, _pack, make_env, seed_for + +META_KEYS = {"model": str, "codec": str, "chunk": int, "execute_steps": int, "obs_offsets": (list, tuple), + "control_hz": (int, float), "upstream": str, "sampler": dict, "ckpt": str} +HAND = 22 + + +def check_meta(meta: dict, fail: list[str]) -> None: + for k, t in META_KEYS.items(): + if k not in meta: + fail.append(f"meta: missing {k!r}") + elif not isinstance(meta[k], t): + fail.append(f"meta: {k!r} is {type(meta[k]).__name__}, expected {t}") + if fail: + return + if not 0 < meta["execute_steps"] <= meta["chunk"]: + fail.append(f"meta: execute_steps {meta['execute_steps']} not in 1..chunk ({meta['chunk']})") + if not meta["obs_offsets"] or any(int(o) > 0 for o in meta["obs_offsets"]) or 0 not in map(int, meta["obs_offsets"]): + fail.append(f"meta: obs_offsets {meta['obs_offsets']} must be <= 0 and include 0") + for name, hw in (meta.get("cameras") or {}).items(): + if len(hw) != 2: + fail.append(f"meta: cameras[{name!r}] = {hw}, expected [H, W]") + + +def check_response(r: dict, meta: dict, state: dict, max_jump: float, tag: str, fail: list[str]) -> None: + h = int(meta["chunk"]) + for k, shape in (("pos", (h, 2, 3)), ("rot", (h, 2, 3, 3)), ("hand", (h, 2, HAND))): + if k not in r: + fail.append(f"{tag}: missing {k!r}") + continue + a = np.asarray(r[k]) + if a.shape != shape: + fail.append(f"{tag}: {k} shape {a.shape}, expected {shape} (chunk={h})") + elif not np.isfinite(a).all(): + fail.append(f"{tag}: {k} has non-finite values") + if fail: + return + rot = np.asarray(r["rot"], np.float64) + ortho = np.abs(rot @ np.swapaxes(rot, -1, -2) - np.eye(3)).max() + det = np.linalg.det(rot) + if ortho > 1e-3 or np.abs(det - 1).max() > 1e-3: + fail.append(f"{tag}: rot not a rotation (max |R R^T - I| {ortho:.2e}, det range {det.min():.4f}..{det.max():.4f})") + if r.get("frame", "ego_camera") != state.get("frame", "ego_camera"): + fail.append(f"{tag}: frame {r.get('frame')!r} != observation frame {state.get('frame')!r}") + jump = np.linalg.norm(np.asarray(r["pos"])[0] - np.asarray(state["pos"])[0], axis=-1).max() + if jump > max_jump: # absolute poses are expected: a large first-step jump usually means a delta was returned + fail.append(f"{tag}: first action is {jump:.3f} m from the current wrist pose (> {max_jump} m); " + "pos must be ABSOLUTE in the observation frame") + if "raw" not in r: + fail.append(f"{tag}: missing 'raw' (model output before decode, logged for debugging)") + + +def main() -> None: + p = argparse.ArgumentParser() + p.add_argument("--port", type=int, required=True) + p.add_argument("--scene", default=sorted(TASKS)[0]) + p.add_argument("--max-jump", type=float, default=0.30, help="max first-step wrist distance from the state (m)") + a = p.parse_args() + + fail: list[str] = [] + policy = PolicyClient(a.port) + meta = policy.meta + print(f"[meta] {meta}", flush=True) + check_meta(meta, fail) + if fail: + sys.exit("FAIL\n " + "\n ".join(fail)) + + env = make_env(meta, a.scene) + offsets = sorted(int(o) for o in meta["obs_offsets"]) + obs0 = _pack(env.reset(seed_for(a.scene, 0))) + req = {"obs": {o: obs0 for o in offsets}, "task": TASKS[a.scene]["task"], "seed": 0, "reset": True, + "prev_cmd": None, "executed": None} + lat = [] + for i in range(2): + t0 = time.perf_counter() + r = policy(req) + lat.append(time.perf_counter() - t0) + check_response(r, meta, req["obs"][0]["state"], a.max_jump, f"request {i}", fail) + if fail: + break + n = int(meta["execute_steps"]) # follow-up as the runner sends it after executing n actions + last = {k: np.asarray(r[k])[n - 1:n] for k in ("pos", "rot", "hand")} + req = dict(req, reset=False, seed=1, prev_cmd=dict(last, frame=obs0["state"].get("frame", "ego_camera")), + executed=n) + print(f"[latency] {', '.join(f'{x:.2f}s' for x in lat)} (first includes warm-up)", flush=True) + if fail: + sys.exit("FAIL\n " + "\n ".join(fail)) + print(f"PASS {meta['model']}: chunk {meta['chunk']}, execute {meta['execute_steps']}, offsets {offsets}") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/sharpa_webxr/compare_judge.py b/benchmarks/sharpa_webxr/compare_judge.py new file mode 100644 index 00000000..a2b752f7 --- /dev/null +++ b/benchmarks/sharpa_webxr/compare_judge.py @@ -0,0 +1,89 @@ +"""Verifier vs VLM judge on policy rollouts: per category / scene confusion between verifier.json (success_any or +sim_prefix_ok) and judge__r2.json (success), with the disagreeing rollout dirs listed for review. + +usage: compare_judge.py [--model gpt-6-luna] [--show N] [--tsv out.tsv] +""" +from __future__ import annotations + +import argparse +import json +from collections import defaultdict +from pathlib import Path + +HERE = Path(__file__).resolve().parent +CATS = json.loads((HERE / "task_categories.json").read_text()) + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("folder", type=Path) + p.add_argument("--model", default="gpt-6-luna") + p.add_argument("--show", type=int, default=3, help="disagreements to list per scene") + p.add_argument("--tsv", type=Path) + a = p.parse_args() + rows = [] + for vf in a.folder.rglob("verifier.json"): + d = vf.parent + jf = d / f"judge_{a.model}_r2.json" + if not jf.exists(): + continue + v, j = json.loads(vf.read_text()), json.loads(jf.read_text()) + if "error" in v or "error" in j: + continue + rows.append({"dir": str(d), "scene": v["scene_id"], "cat": CATS.get(v["scene_id"], "?"), "needs_judge": v["needs_judge"], + "ver": bool(v["sim_prefix_ok"]), "ver_any": bool(v.get("sim_prefix_any_time", v["sim_prefix_ok"])), + "ver_end": bool(v.get("sim_prefix_end_state", v["sim_prefix_ok"])), "judge": bool(j["success"]), "judge_conf": j.get("confidence"), + "stages": [(s["name"], s["achieved"]) for s in v["stages"]], "rationale": j.get("rationale", "")[:160], + "j_stages": {k: j.get(k) for k in ("reach", "grasp", "move", "complete")}}) + by_cat, by_scene = defaultdict(list), defaultdict(list) + for r in rows: + by_cat[r["cat"]].append(r) + by_scene[r["scene"]].append(r) + + def conf(rs): + n = len(rs) + both = sum(r["ver"] and r["judge"] for r in rs) + vonly = sum(r["ver"] and not r["judge"] for r in rs) + jonly = sum(r["judge"] and not r["ver"] for r in rs) + neither = n - both - vonly - jonly + return n, both, vonly, jonly, neither + + print(f"{'category':26s} {'n':>4s} {'ver_SR':>7s} {'judge_SR':>8s} {'agree':>6s} {'both':>5s} {'ver_only':>8s} {'judge_only':>10s}") + out = [] + for cat, rs in sorted(by_cat.items()): + n, both, vo, jo, ne = conf(rs) + rec = {"category": cat, "n": n, "ver_sr": round(sum(r["ver"] for r in rs) / n, 3), "ver_any_sr": round(sum(r["ver_any"] for r in rs) / n, 3), + "ver_end_sr": round(sum(r["ver_end"] for r in rs) / n, 3), "judge_sr": round(sum(r["judge"] for r in rs) / n, 3), + "agree": round((both + ne) / n, 3), "both": both, "ver_only": vo, "judge_only": jo} + out.append(rec) + print(f"{cat:26s} {n:4d} {rec['ver_sr']:7.3f} {rec['judge_sr']:8.3f} {rec['agree']:6.3f} {both:5d} {vo:8d} {jo:10d} any={rec['ver_any_sr']:.3f} end={rec['ver_end_sr']:.3f}") + n, both, vo, jo, ne = conf(rows) + print(f"{'ALL':26s} {n:4d} {sum(r['ver'] for r in rows) / n:7.3f} {sum(r['judge'] for r in rows) / n:8.3f} {(both + ne) / n:6.3f} {both:5d} {vo:8d} {jo:10d}") + sim_rows = [r for r in rows if not r["needs_judge"]] + n, both, vo, jo, ne = conf(sim_rows) + print(f"{'SIM-ONLY scenes':26s} {n:4d} {sum(r['ver'] for r in sim_rows) / n:7.3f} {sum(r['judge'] for r in sim_rows) / n:8.3f} {(both + ne) / n:6.3f} {both:5d} {vo:8d} {jo:10d}") + # judge-ending scenes: how often the sim prefix gate would have skipped a judge call, and judge successes it would have blocked + jr = [r for r in rows if r["needs_judge"]] + if jr: + blocked = sum(not r["ver"] for r in jr) + blocked_js = sum((not r["ver"]) and r["judge"] for r in jr) + print(f"\njudge-ending scenes: {len(jr)} rollouts; sim prefix fails {blocked} ({blocked / len(jr):.0%}) -> judge calls saved; " + f"of those the judge said success in {blocked_js}") + + print(f"\n{'scene':8s} {'category':24s} {'n':>3s} {'ver':>4s} {'jdg':>4s} {'both':>4s} {'v!j':>4s} {'j!v':>4s}") + for sc, rs in sorted(by_scene.items()): + n, both, vo, jo, ne = conf(rs) + print(f"{sc[:8]:8s} {rs[0]['cat'][:24]:24s} {n:3d} {sum(r['ver'] for r in rs):4d} {sum(r['judge'] for r in rs):4d} {both:4d} {vo:4d} {jo:4d}") + for r in [r for r in rs if r["ver"] != r["judge"]][: a.show]: + tag = "VER-only" if r["ver"] else "JUDGE-only" + print(f" {tag:10s} {r['dir']} stages={r['stages']} judge={r['j_stages']} ({r['judge_conf']}) {r['rationale']}") + if a.tsv: + cols = list(out[0].keys()) + with open(a.tsv, "w") as f: + f.write("\t".join(cols) + "\n") + for r in out: + f.write("\t".join(str(r[c]) for c in cols) + "\n") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/sharpa_webxr/env.py b/benchmarks/sharpa_webxr/env.py new file mode 100644 index 00000000..c7594568 --- /dev/null +++ b/benchmarks/sharpa_webxr/env.py @@ -0,0 +1,260 @@ +"""Model-agnostic Sharpa env on the WebXR-Teleop MuJoCo sim (TacoSim, yam_duo): actions in, observations out. + +The env knows nothing about any policy. It takes one action per control step, holds it for 50 ms, and returns the +observation keys it was configured with. How often to observe, how many actions to execute per prediction, and how +to encode inputs belong to the policy server (policies/_server.py), driven by rollout.py. + +Physics, gains, action smoothing, layout sidecar and reset randomization are the teleop app's own +(``mujoco_webxr_teleop.app`` defaults == the settings recorded with simteleop_1004_fleet episodes: +2400 Hz physics, 60 Hz tick, 50 ms action smoothing updated at 2400 Hz, kp1200critical0918 arm gains). +Control runs at 20 Hz: each action is held for 3 ticks, matching the simteleop LeRobot alignment +(frame k = obs/joint_pos[3k], action = obs/joint_setpoints[3k+2]). Reset uses a seeded RNG, so (scene, seed) gives +the same initial state every time. + +Action spaces (``action_space=``): + "joint" [56] absolute joint targets in ``env.joint_names`` order (or a {joint_name: value} dict). + "eef" {"left": {"pose": 4x4, "hand": [22]}, "right": {...}}: wrist pose in ``eef_frame`` plus absolute hand + joints. The env solves arm joints with IK (scripts/sharpa_eef/eef_kinematics.ik), each solve seeded from + the previous one (from the measured joints after reset); IK residuals are returned in ``info``. + +Observation keys (``obs=ObsConfig(...)``): + "joints" [56] measured joint positions (+ "joint_names") + "eef" {"pos": [2,3], "rot": [2,3,3], "hand": [2,22]} wrist pose in ``eef_frame`` (sides left, right) + "object_poses" free-body object poses from the sim + "time" sim time (s) + "images" {camera: HxWx3 uint8} for ``cameras`` (render only when asked: CPU OSMesa, ~0.5-1 s per frame) +""" +from __future__ import annotations + +import os +import sys +from dataclasses import dataclass, field +from pathlib import Path + +os.environ.setdefault("MUJOCO_GL", "osmesa") +os.environ.setdefault("PYOPENGL_PLATFORM", os.environ["MUJOCO_GL"]) + +import mujoco # noqa: E402 +import numpy as np # noqa: E402 + +REPO = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(REPO / "third_party" / "WebXR-Teleop")) +sys.path.insert(0, str(REPO / "scripts/sharpa_eef")) +from mujoco_webxr_teleop import app as teleop_app # noqa: E402 +from mujoco_webxr_teleop.taco.sim import Gains, SimRates, TacoSim # noqa: E402 + +import canonical as C # noqa: E402 +import eef_kinematics as K # noqa: E402 + +CONTROL_HZ = 20.0 +TICKS_PER_ACTION = 3 # 60 Hz teleop ticks per 20 Hz control step + +# Dataset cameras (assets/sharpa_full_meta/*/meta/source.json): constant base_from_camera in the robot base frame, +# optical frame (x right, y down, z forward), recorded at 640x480. Pass `cameras=` to add or override. +CAMERAS = { + "ego_view": {"pos": (0.0, 0.0, 0.6), "pitch_down_deg": 55.0, "fovy_deg": 60.0}, + "chest_view": {"pos": (0.0, 0.0, 0.25), "pitch_down_deg": 30.0, "fovy_deg": 90.0}, +} +IMAGE_HW = (480, 640) +STATE_KEYS = ("joints", "eef", "object_poses", "time") + + +@dataclass +class ObsConfig: + """What the env returns. ``cameras`` maps camera name -> (H, W) (a list means the native 480x640); + ``state`` lists keys from STATE_KEYS.""" + cameras: dict = field(default_factory=lambda: {v: IMAGE_HW for v in CAMERAS}) + state: tuple = ("joints", "eef", "object_poses", "time") + + def __post_init__(self): + if isinstance(self.cameras, (list, tuple)): + self.cameras = {v: IMAGE_HW for v in self.cameras} + self.cameras = {v: tuple(hw) for v, hw in self.cameras.items()} + bad = set(self.state) - set(STATE_KEYS) + if bad: + raise ValueError(f"unknown state keys {bad}; choose from {STATE_KEYS}") + + def keys(self) -> list[str]: + return list(self.state) + (["images"] if self.cameras else []) + + +def base_from_camera(view: str, cameras: dict | None = None) -> np.ndarray: + """4x4 base_from_camera, identical to the dataset's camera_pose. columns.""" + c = (cameras or CAMERAS)[view] + p = np.deg2rad(c["pitch_down_deg"]) + fwd = np.array([np.cos(p), 0.0, -np.sin(p)]) + right = np.array([0.0, -1.0, 0.0]) + down = np.cross(fwd, right) + T = np.eye(4) + T[:3, :3] = np.stack([right, down, fwd], axis=1) + T[:3, 3] = c["pos"] + return T + + +def _teleop_args(scene_xml: Path): + parser = teleop_app.build_parser() + args = parser.parse_args(["--embodiment", "yam_duo", "--scene_xml", str(scene_xml)]) + defaults = {k: parser.get_default(k) for k in + ("dr_object_xy", "dr_object_yaw", "dr_arm_jitter", "dr_object_bias", "settle_time")} + return args, defaults + + +class SharpaWebXREnv: + """reset(seed) -> obs; step(action, observe=True) -> (obs | None, info).""" + + def __init__(self, scene_dir: Path, obs: ObsConfig | None = None, action_space: str = "joint", + eef_frame: str = "ego_view", cameras: dict | None = None): + """``eef_frame``: "arm_base" (each arm's own base) or a camera name (that camera's optical frame; the + dataset's canonical frame is "ego_view").""" + if action_space not in ("joint", "eef"): + raise ValueError(action_space) + self.scene_dir = Path(scene_dir) + self.scene_xml = self.scene_dir / "scene.xml" + self.obs_config = obs or ObsConfig() + self.action_space = action_space + self.cameras = {**CAMERAS, **(cameras or {})} + unknown = set(self.obs_config.cameras) - set(self.cameras) + if unknown: + raise ValueError(f"unknown cameras {unknown}; pass their extrinsics with cameras=") + self.eef_frame = eef_frame + self._cam_from_arm = None + if eef_frame != "arm_base": + self._bfc_eef = base_from_camera(eef_frame, self.cameras) + self._cam_from_arm = C.camera_from_arm_base(self._bfc_eef) + self._arm_from_cam = {s: np.linalg.inv(T) for s, T in self._cam_from_arm.items()} + + args, self._defaults = _teleop_args(self.scene_xml) + self._args = args + rates = SimRates(args.physics_hz, args.control_hz) + gains = Gains(args.arm_kp, args.arm_kd, args.hand_kp, args.hand_kd, arm_profile=args.arm_gain_profile) + self.sim = TacoSim(rates, args.embodiment, gains, None, scene_xml=self.scene_xml, + retarget_backend=args.hand_ik, detached_hands=False, arm_mink=args.arm_ik == "mink", + arm_full_range=None, kinematic_arm=args.kinematic_arm, + action_smoothing_ms=args.action_smoothing_ms, action_update_hz=args.action_update_hz) + assert abs(rates.control_hz - CONTROL_HZ * TICKS_PER_ACTION) < 1e-9, rates + m = self.sim.model + self.robot_act = np.asarray(self.sim._robot_act) # noqa: SLF001 + self.ctrl_lo, self.ctrl_hi = m.actuator_ctrlrange[self.robot_act].T + self.joint_names = [mujoco.mj_id2name(m, mujoco.mjtObj.mjOBJ_JOINT, int(m.actuator_trnid[a, 0])) + for a in self.robot_act] + self._jidx = {n: i for i, n in enumerate(self.joint_names)} + self._cols = C.joint_columns(self.joint_names) + self._renderers = {} # (h, w) -> Renderer; one OSMesa context per size, built once + self._ik_seed = None + self.step_count = 0 + self.last_target = None + + # ---- episode control ------------------------------------------------------------------- + def reset(self, seed: int) -> dict: + """Teleop reset (layout sidecar, arm jitter, object xy/yaw randomization, settle) with a seeded RNG.""" + rng = np.random.default_rng(seed) + teleop_app.prepare_scene(self.sim, self.scene_xml, self._args, rng, self._defaults) + self.randomization = dict(self.sim.recording_randomization) + self.step_count = 0 + q = self.joints() + self._ik_seed = {s: q[self._cols[s][0]].copy() for s in C.SIDES} + return self.observe() + + def step(self, action, observe=True): + """Apply one action for one 50 ms control step. ``observe``: True (the configured keys), False (skip and + return None, e.g. between replans), or a list of keys for this step only. Returns (obs, info).""" + q, info = self._to_joint_target(action) + q = np.clip(q, self.ctrl_lo, self.ctrl_hi) + sim = self.sim + if sim.action_smoothing_ms: + sim._action_target[self.robot_act] = q # noqa: SLF001 same field teleop's tick writes + else: + sim.data.ctrl[self.robot_act] = q + for _ in range(TICKS_PER_ACTION): + sim.tick({}) + self.step_count += 1 + self.last_target = q + if observe is False: + return None, info + return self.observe(None if observe is True else observe), info + + def _to_joint_target(self, action): + if self.action_space == "joint": + if isinstance(action, dict): + q = self.joints() if self.last_target is None else self.last_target.copy() + for n, v in action.items(): + q[self._jidx[n]] = v + return q, {} + q = np.asarray(action, np.float64) + if q.shape != (len(self.joint_names),): + raise ValueError(f"joint action must be [{len(self.joint_names)}] in env.joint_names order") + return q, {} + q = np.zeros(len(self.joint_names)) + info = {"ik_pos_err": np.zeros(2), "ik_rot_err": np.zeros(2), "ik_converged": np.zeros(2, bool)} + for k, s in enumerate(C.SIDES): + arm, hcols = self._cols[s] + T = np.asarray(action[s]["pose"], np.float64) + if self._cam_from_arm is not None: + T = self._arm_from_cam[s] @ T + qs, pe, re, ok = K.ik(s, T[None], self._ik_seed[s][None]) + self._ik_seed[s] = qs[0] + q[arm] = qs[0] + q[hcols] = action[s]["hand"] + info["ik_pos_err"][k], info["ik_rot_err"][k], info["ik_converged"][k] = pe[0], re[0], ok[0] + return q, info + + # ---- observation ----------------------------------------------------------------------- + def joints(self) -> np.ndarray: + d, m = self.sim.data, self.sim.model + return np.array([d.qpos[m.jnt_qposadr[int(m.actuator_trnid[a, 0])]] for a in self.robot_act]) + + def eef(self, joints: np.ndarray | None = None) -> C.Canonical: + """Wrist poses + hand joints in ``eef_frame`` (a Canonical with N=1).""" + q = self.joints() if joints is None else joints + if self._cam_from_arm is None: + return C.from_joints(q[None], self.joint_names, frame="arm_base") + return C.from_joints(q[None], self.joint_names, frame="ego_camera", base_from_camera=self._bfc_eef.ravel()) + + def observe(self, keys=None) -> dict: + keys = self.obs_config.keys() if keys is None else keys + obs = {} + q = self.joints() + if "joints" in keys: + obs["joints"], obs["joint_names"] = q, self.joint_names + if "eef" in keys: + e = self.eef(q) + obs["eef"] = {"pos": e.pos[0], "rot": e.rot[0], "hand": e.hand[0], "frame": self.eef_frame, "canonical": e} + if "object_poses" in keys: + obs["object_poses"] = self.sim.object_poses() + if "time" in keys: + obs["time"] = self.sim.data.time + if "images" in keys: + obs["images"] = {v: self.render(v, hw) for v, hw in self.obs_config.cameras.items()} + return obs + + def world_from_base(self) -> np.ndarray: + pos, quat = self.sim.root_pose() + T = np.eye(4) + mat = np.empty(9) + mujoco.mju_quat2Mat(mat, np.asarray(quat, np.float64)) + T[:3, :3] = mat.reshape(3, 3) + T[:3, 3] = pos + return T + + def _get_renderer(self, hw): + if hw not in self._renderers: + self._renderers[hw] = mujoco.Renderer(self.sim.model, *hw) + return self._renderers[hw] + + def render(self, view: str, hw: tuple[int, int] | None = None) -> np.ndarray: + T = self.world_from_base() @ base_from_camera(view, self.cameras) + fwd, pos = T[:3, 2], T[:3, 3] + cam = mujoco.MjvCamera() + cam.type = mujoco.mjtCamera.mjCAMERA_FREE + cam.distance = 1.0 + cam.lookat[:] = pos + fwd * cam.distance + cam.azimuth = float(np.degrees(np.arctan2(fwd[1], fwd[0]))) + cam.elevation = float(np.degrees(np.arcsin(np.clip(fwd[2], -1, 1)))) + m = self.sim.model + old = float(m.vis.global_.fovy) + m.vis.global_.fovy = self.cameras[view]["fovy_deg"] + r = self._get_renderer(tuple(hw) if hw is not None else IMAGE_HW) + r.update_scene(self.sim.data, camera=cam) + img = r.render().copy() + m.vis.global_.fovy = old + return img diff --git a/benchmarks/sharpa_webxr/judge.py b/benchmarks/sharpa_webxr/judge.py new file mode 100644 index 00000000..933e4359 --- /dev/null +++ b/benchmarks/sharpa_webxr/judge.py @@ -0,0 +1,119 @@ +"""VLM judge for Sharpa sim rollouts: Claude Opus 5.5 watches 16 frames (ego | chest views) of video.mp4. + +GWP-0.5 graded rubric (report Sec. 4): 0.25 each for reach, grasp, move-to-target, place/complete; plus binary +success ("task completed at any point", RoboTwin semantics). Sim signals (signals.json) decide reach/grasp/move +when they fire; the judge decides place/complete and success. Output: judge.json next to the video. + +Credentials: ANTHROPIC_API_KEY / ANTHROPIC_AUTH_TOKEN / `ant auth login` profile. +""" +from __future__ import annotations + +import argparse +import base64 +import io +import json +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +import anthropic +import imageio.v3 as iio +from PIL import Image + +MODEL = "claude-opus-5-5" +N_FRAMES = 16 + +RUBRIC = """You grade robot manipulation rollouts from a physics simulator. A bimanual robot (two arms with +five-fingered dexterous hands) is given a natural-language task. You see frames sampled uniformly in time from one +rollout. Each frame shows two cameras side by side: left half = head camera looking down at the table, right +half = chest camera looking forward across the table. Frame captions give the time in seconds. + +Score four stages, each true or false, judging only what is visible: +1. reach: a hand gets to (touches or is within a few cm of) the object the task is about. +2. grasp: the hand(s) take hold of or firmly engage that object (grasp, pinch, push on a lid/handle as the task + requires). Incidental brushing is not a grasp. +3. move: the object is moved purposefully toward the task goal (carried, rotated, opened/closed partially, poured...). +4. complete: the task's goal state is reached (e.g. object placed in/on the target, lid closed, cap on). For + tasks without a placement (shake, stir, toss, hit), complete means the described action was clearly performed. + +success = the full task was accomplished at any time in the rollout (even if disturbed later). +Be strict: if the evidence is ambiguous, answer false. Knocking objects off the table, or the goal object ending +in an implausible state, is not success. Later stages normally imply earlier ones; flag contradictions in notes.""" + +SCHEMA = { + "type": "object", + "properties": { + "reach": {"type": "boolean"}, "grasp": {"type": "boolean"}, "move": {"type": "boolean"}, + "complete": {"type": "boolean"}, "success": {"type": "boolean"}, + "confidence": {"type": "string", "enum": ["low", "medium", "high"]}, + "rationale": {"type": "string"}, + }, + "required": ["reach", "grasp", "move", "complete", "success", "confidence", "rationale"], + "additionalProperties": False, +} + + +def sample_frames(video: Path, n: int = N_FRAMES, fps: float = 10.0): + frames = iio.imread(video, plugin="pyav") + idx = [round(i * (len(frames) - 1) / (n - 1)) for i in range(n)] + out = [] + for i in idx: + buf = io.BytesIO() + Image.fromarray(frames[i]).save(buf, format="JPEG", quality=85) + out.append((i / fps, base64.standard_b64encode(buf.getvalue()).decode())) + return out + + +def keyframes(rollout_dir: Path): + """Pre-rendered keyframes (postprocess --keyframes) if present, else sampled from video.mp4.""" + kd = rollout_dir / "keyframes" + if kd.is_dir() and any(kd.glob("t*.jpg")): + return [(float(f.stem[1:]), base64.standard_b64encode(f.read_bytes()).decode()) for f in sorted(kd.glob("t*.jpg"))] + return sample_frames(rollout_dir / "video.mp4") + + +def judge_one(client: anthropic.Anthropic, rollout_dir: Path, force: bool = False) -> dict: + out = rollout_dir / "judge.json" + if out.exists() and not force: + return json.loads(out.read_text()) + meta = json.loads((rollout_dir / "rollout.json").read_text()) + content = [{"type": "text", "text": f"Task: {meta['task']}\nRollout length: {meta['steps'] / 20:.1f} s."}] + for t, b64 in keyframes(rollout_dir): + content += [{"type": "text", "text": f"t = {t:.1f} s"}, + {"type": "image", "source": {"type": "base64", "media_type": "image/jpeg", "data": b64}}] + content.append({"type": "text", "text": "Grade this rollout."}) + resp = client.beta.messages.create( + model=MODEL, + max_tokens=16000, + betas=["server-side-fallback-2026-07-01"], + fallbacks="default", + system=[{"type": "text", "text": RUBRIC, "cache_control": {"type": "ephemeral"}}], + output_config={"effort": "medium", "format": {"type": "json_schema", "schema": SCHEMA}}, + messages=[{"role": "user", "content": content}], + ) + if resp.stop_reason == "refusal": + verdict = {"error": "refusal", "stop_details": str(resp.stop_details)} + else: + text = next(b.text for b in resp.content if b.type == "text") + verdict = json.loads(text) + verdict.update({"model": resp.model, "request_id": resp._request_id, "frames": N_FRAMES, + "usage": {"input": resp.usage.input_tokens, "output": resp.usage.output_tokens, + "cache_read": resp.usage.cache_read_input_tokens}}) + out.write_text(json.dumps(verdict, indent=1)) + return verdict + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("dirs", nargs="+", type=Path) + p.add_argument("--workers", type=int, default=8) + p.add_argument("--force", action="store_true") + a = p.parse_args() + client = anthropic.Anthropic() + dirs = [d for d in a.dirs if (d / "video.mp4").exists() or (d / "keyframes").is_dir()] + with ThreadPoolExecutor(a.workers) as ex: + for d, v in zip(dirs, ex.map(lambda d: judge_one(client, d, a.force), dirs)): + print(d, {k: v.get(k) for k in ("reach", "grasp", "move", "complete", "success", "confidence", "error")}, flush=True) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/sharpa_webxr/limits.py b/benchmarks/sharpa_webxr/limits.py new file mode 100644 index 00000000..8ea5d2b5 --- /dev/null +++ b/benchmarks/sharpa_webxr/limits.py @@ -0,0 +1,18 @@ +"""Episode length limit, shared by the runner and every offline pass (signals, judge frames, videos). + +limit(scene) = min(ceil(1.5 x p90 demo length), MAX_SECONDS), in 20 Hz control steps. Policies are causal, so a +rollout recorded with a longer limit is cut to this one offline without re-running it (the first N steps are +exactly what a run capped at N would have produced). +""" +from __future__ import annotations + +import json +from pathlib import Path + +CONTROL_HZ = 20 +MAX_SECONDS = 20.0 +TASKS = json.loads((Path(__file__).resolve().parent / "tasks.json").read_text()) + + +def limit(scene_id: str, max_seconds: float = MAX_SECONDS) -> int: + return min(int(TASKS[scene_id]["max_steps"]), int(round(max_seconds * CONTROL_HZ))) diff --git a/benchmarks/sharpa_webxr/make_gt_rollouts.py b/benchmarks/sharpa_webxr/make_gt_rollouts.py new file mode 100644 index 00000000..a7ba0851 --- /dev/null +++ b/benchmarks/sharpa_webxr/make_gt_rollouts.py @@ -0,0 +1,39 @@ +"""Ground-truth "rollouts" from simteleop teleop recordings (gs://.../webxr-teleop/live/episodes): recorded qpos at +20 Hz (every 3rd 60 Hz row) in the rollout format, so postprocess/judge can be validated on known outcomes.""" +import argparse +import json +import subprocess +import tempfile +from pathlib import Path + +import h5py +import numpy as np + +LIVE = "gs://foundational-research/webxr-teleop/live" +SRC = Path("assets/sharpa_full_meta/simteleop_1004_fleet/meta/source.json") + +p = argparse.ArgumentParser() +p.add_argument("--scenes", nargs="+", required=True) +p.add_argument("--per-scene", type=int, default=2) +p.add_argument("--out", type=Path, required=True) +p.add_argument("--split", type=Path, default=Path("assets/sharpa_full_meta/split_v1.json")) +a = p.parse_args() +eps = json.loads(SRC.read_text())["episodes"] +val = set(json.loads(a.split.read_text())["sources"]["simteleop_1004_fleet"]["val_episodes"]) +for sc in a.scenes: + cand = [e for e in eps if e["scene_id"] == sc] + cand = sorted(cand, key=lambda e: (e["episode_index"] not in val, e["episode_index"]))[: a.per_scene] + for e in cand: + d = a.out / sc / f"gt_ep{e['episode_index']}" + if (d / "rollout.json").exists(): + continue + with tempfile.TemporaryDirectory() as tmp: + subprocess.run(["gsutil", "-q", "cp", f"{LIVE}/{e['upload_prefix']}/trajectory.hdf5", tmp], check=True) + g = h5py.File(Path(tmp) / "trajectory.hdf5")["data/demo_0"] + q = np.concatenate([g["initial_state/qpos"][:][None], g["states/qpos"][2::3]]) + d.mkdir(parents=True, exist_ok=True) + np.savez_compressed(d / "trajectory.npz", qpos=q) + (d / "rollout.json").write_text(json.dumps({"scene_id": sc, "seed": f"gt_ep{e['episode_index']}", "gt": True, + "episode_index": e["episode_index"], "val": e["episode_index"] in val, + "outcome": e.get("outcome"), "task": e["task"], "steps": len(q) - 1})) + print(d, len(q), e.get("outcome"), "val" if e["episode_index"] in val else "train") diff --git a/benchmarks/sharpa_webxr/policies/__init__.py b/benchmarks/sharpa_webxr/policies/__init__.py new file mode 100644 index 00000000..710b3ba3 --- /dev/null +++ b/benchmarks/sharpa_webxr/policies/__init__.py @@ -0,0 +1,6 @@ +"""Per-model policy servers on the policies/common.py contract (each runs in the training .venv, on a GPU). + +A server owns its model's whole I/O (image layout, state encoding, normalization, sampler, action decode) and how +it is deployed upstream (execute_steps, past-observation rows), and publishes both through its "meta" endpoint. +The sim side (rollout.py) is the same for every model. +""" diff --git a/benchmarks/sharpa_webxr/policies/common.py b/benchmarks/sharpa_webxr/policies/common.py new file mode 100644 index 00000000..cf6d9ba3 --- /dev/null +++ b/benchmarks/sharpa_webxr/policies/common.py @@ -0,0 +1,105 @@ +"""Shared contract between rollout.py (sim side, .venv_webxr) and the per-model policy servers (.venv). + +Every server owns its model's whole I/O: image layout, state encoding, normalization, sampler, and the action +codec decode. The sim side only sends the measured canonical state and camera images and receives absolute +canonical wrist poses + hand joints, so the rollout loop and IK are identical for every model. + +Endpoints (giga_models ZeroMQ RobotInferenceServer, torch-serialized dicts): + "meta" -> {"model", "codec", "chunk", "execute_steps", "obs_offsets", "control_hz", "upstream", "sampler", + optional "cameras": {name: [H, W]} (default ego_view + chest_view at 480x640)} + execute_steps: actions run before replanning, matched to the model's upstream deploy. + obs_offsets: rows (<= 0) at which the policy needs past observations, e.g. [-5, 0] for LDA-1B. + "inference" <- {"obs": {offset: {"images": {"ego_view": uint8 [480,640,3], "chest_view": ...}, + "state": canonical dict (1 row)}}, # one entry per meta obs_offsets + "task": str, "seed": int, "reset": bool, # reset=True on the first chunk + "prev_cmd": canonical dict (1 row) | None, # last commanded pose (LDA anchor) + "executed": int | None} # actions run since the last call + -> {"pos": [H,2,3], "rot": [H,2,3,3], "hand": [H,2,22] (absolute, ego-camera frame), + "raw": float32 [H, width] model output before decode, "timing": {...}, "ckpt": str} +""" +from __future__ import annotations + +import sys +import time +from pathlib import Path + +import numpy as np + +REPO = Path(__file__).resolve().parents[3] +for p in (REPO / "scripts/sharpa_eef", REPO / "third_party/giga-world-policy/third_party/giga-models"): + if str(p) not in sys.path: + sys.path.insert(0, str(p)) + +import canonical as C # noqa: E402 + + +def canon_to_dict(c: C.Canonical) -> dict: + return {"pos": np.asarray(c.pos, np.float64), "rot": np.asarray(c.rot, np.float64), + "hand": np.asarray(c.hand, np.float64), "frame": c.frame} + + +def dict_to_canon(d: dict | None) -> C.Canonical | None: + if d is None: + return None + pos = np.asarray(d["pos"], np.float64) + return C.Canonical(pos, np.asarray(d["rot"], np.float64), np.asarray(d["hand"], np.float64), + np.ones(len(pos), bool), frame=d.get("frame", "ego_camera")) + + +class BasePolicy: + """Subclass: set the meta fields and implement ``act``. ``reset`` clears per-episode state.""" + + model = "" + codec = "" + chunk = 0 + execute_steps = 0 + obs_offsets = (0,) + control_hz = 20.0 + upstream = "" # which upstream deploy path the settings copy + sampler = {} # e.g. {"name": "unipc", "order": 2, "steps": 30, "shift": 5.0, "cfg": 1.0} + ckpt = "" + + def meta(self) -> dict: + return {k: getattr(self, k) for k in ("model", "codec", "chunk", "execute_steps", "obs_offsets", + "control_hz", "upstream", "sampler", "ckpt")} + + def reset(self) -> None: + pass + + def act(self, obs: dict, task: str, seed: int, prev_cmd: C.Canonical | None) -> tuple[C.Canonical, np.ndarray]: + """obs: {offset: {"images": {...}, "state": Canonical}} -> (absolute Canonical [H], raw [H, width]).""" + raise NotImplementedError + + def inference(self, data: dict) -> dict: + t0 = time.perf_counter() + if data.get("reset"): + self.reset() + obs = {int(k): {"images": v["images"], "state": dict_to_canon(v["state"])} for k, v in data["obs"].items()} + cmd, raw = self.act(obs, data["task"], int(data.get("seed", 0)), dict_to_canon(data.get("prev_cmd"))) + try: + import torch + if torch.cuda.is_available(): + torch.cuda.synchronize() + except ImportError: + pass + out = canon_to_dict(cmd) + out.update(raw=np.asarray(raw, np.float32), ckpt=self.ckpt, + timing={"server_ms": (time.perf_counter() - t0) * 1e3}) + return out + + +def cap_gpu_memory(gib: float) -> None: + """Cap this process's CUDA caching allocator so an OOM hits the eval server, not a co-located training job.""" + import torch + if torch.cuda.is_available(): + total = torch.cuda.get_device_properties(0).total_memory + torch.cuda.set_per_process_memory_fraction(min(1.0, gib * 2**30 / total), 0) + + +def serve(policy: BasePolicy, port: int) -> None: + from giga_models.sockets.server import RobotInferenceServer + + srv = RobotInferenceServer(policy, port=port) + srv.register_endpoint("meta", policy.meta, requires_input=False) + print(f"[serve] {policy.model} on :{port} meta={policy.meta()}", flush=True) + srv.run() diff --git a/benchmarks/sharpa_webxr/policies/gwp_server.py b/benchmarks/sharpa_webxr/policies/gwp_server.py new file mode 100644 index 00000000..1ff301e9 --- /dev/null +++ b/benchmarks/sharpa_webxr/policies/gwp_server.py @@ -0,0 +1,60 @@ +"""GWP-0.5 policy server on the policies/common.py contract (runs in .venv). + +Model and sampler are policy_server.GWPSharpaPolicy unchanged (upstream action-only loop: FlowMatchEuler shift 5, +10 steps, no CFG, prefix KV cache over state + reference frame). Upstream-matched replanning: upstream executes 30 of +its 48-action chunk (run_inference_openloop.sh REPLAN_STEPS=30, 30 Hz) = 62.5% = 1.0 s; ours executes 20 of 32 +(20 Hz) = 62.5% = 1.0 s. +""" +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE.parent)) +sys.path.insert(0, str(HERE)) +from common import BasePolicy, cap_gpu_memory, serve # noqa: E402 + +import action_codecs as A # noqa: E402 + + +class GWPServer(BasePolicy): + model = "gwp05" + codec = "giga" + chunk = 32 + execute_steps = 20 + obs_offsets = (0,) + upstream = "giga-world-policy scripts/inference_openloop.py + run_inference_openloop.sh (REPLAN_STEPS=30 of 48)" + + def __init__(self, ckpt: str, num_steps: int = 10, prep_dir: str | None = None): + from policy_server import PREP_DIR, GWPSharpaPolicy + + self.p = GWPSharpaPolicy(ckpt, num_steps=num_steps, prep_dir=Path(prep_dir) if prep_dir else PREP_DIR) + self.ckpt = ckpt + self.sampler = {"name": "flowmatch_euler", "steps": num_steps, "shift": 5.0, "cfg": None, + "video": "not generated (action-only, prefix KV cache)"} + self.C = A.CODECS["giga"] + + def act(self, obs, task, seed, prev_cmd): + o = obs[0] + state62 = self.C.state_encode(o["state"])[0] + raw = self.p.predict(o["images"], state62, task, seed) + return self.C.decode(raw, state=o["state"]), raw + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--ckpt", required=True) + ap.add_argument("--port", type=int, default=11700) + ap.add_argument("--num-steps", type=int, default=10) + ap.add_argument("--max-gpu-gib", type=float, default=0, help="cap allocator (0 = no cap)") + ap.add_argument("--prep-dir", default=None, help="training prep dir (norm_stats.json, t5_task_embeddings.pt)") + a = ap.parse_args() + if a.max_gpu_gib: + cap_gpu_memory(a.max_gpu_gib) + serve(GWPServer(a.ckpt, a.num_steps, a.prep_dir), a.port) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/sharpa_webxr/policies/hold_server.py b/benchmarks/sharpa_webxr/policies/hold_server.py new file mode 100644 index 00000000..7a0048c7 --- /dev/null +++ b/benchmarks/sharpa_webxr/policies/hold_server.py @@ -0,0 +1,44 @@ +"""Reference policy server: holds the current pose. No model, no GPU. + +The smallest complete implementation of the policies/common.py contract, for two uses: +- run the whole eval end to end (rollout.py, postprocess.py, score.py) without any model; +- a template for a new model's server: subclass BasePolicy, set the meta fields, implement ``act``. + +usage: python benchmarks/sharpa_webxr/policies/hold_server.py --port 11600 +""" +from __future__ import annotations + +import argparse + +import numpy as np + +from common import BasePolicy, C, serve + + +class HoldPolicy(BasePolicy): + model = "hold" + codec = "none" + chunk = 16 # actions returned per request + execute_steps = 16 # actions the runner executes before calling again + obs_offsets = (0,) # observation rows needed, relative to now (e.g. (-5, 0) for a t-5 history) + control_hz = 20.0 + upstream = "reference implementation (no upstream)" + sampler = {"name": "none"} + ckpt = "none" + + def act(self, obs: dict, task: str, seed: int, prev_cmd: C.Canonical | None) -> tuple[C.Canonical, np.ndarray]: + s = obs[0]["state"] # canonical state of the current row: pos [1,2,3], rot [1,2,3,3], hand [1,2,22] + h = self.chunk + cmd = C.Canonical(np.repeat(s.pos, h, 0), np.repeat(s.rot, h, 0), np.repeat(s.hand, h, 0), + np.ones(h, bool), frame=s.frame) + return cmd, np.zeros((h, 1), np.float32) # raw: the model's undecoded output (logged only) + + +if __name__ == "__main__": + p = argparse.ArgumentParser() + p.add_argument("--port", type=int, default=11600) + p.add_argument("--ckpt", default="none", help="ignored (no weights); accepted so watchers can start it like any server") + a = p.parse_args() + pol = HoldPolicy() + pol.ckpt = a.ckpt + serve(pol, a.port) diff --git a/benchmarks/sharpa_webxr/policy_server.py b/benchmarks/sharpa_webxr/policy_server.py new file mode 100644 index 00000000..fcb40bc3 --- /dev/null +++ b/benchmarks/sharpa_webxr/policy_server.py @@ -0,0 +1,122 @@ +"""GWP-0.5 Sharpa policy server for closed-loop sim eval (runs in the OpenWAM .venv, one per GPU). + +Model side reuses the training script verbatim (scripts/parallel_sharpa_forward_checks/run_giga_sharpa_train_job.py): +TRANSFORMER_CFG (62-D I/O, embodiment 2), SharpaGigaDataset._process_images/_compose_views (ego on top, chest +below, 320x384, val-mode center crop), quantile normalization and ``sample_actions`` (upstream action-only +loop, FlowMatchEuler shift 5, no CFG). Default 10 denoising steps = GWP-0.5 deployment (wam.cpp released +configs); training's decode check used 20. + +Transport is upstream's ZeroMQ RobotInferenceServer (giga_models.sockets), endpoint "inference": + in : {"images": {"ego_view": uint8 [480,640,3], "chest_view": ...}, "state": float [62] (codec state_encode), + "task": str, "seed": int} + out: {"action": float32 [32,62] denormalized giga-codec chunk, "timing": {...}} +""" +from __future__ import annotations + +import argparse +import importlib.util +import json +import os +import sys +import time +import types +from pathlib import Path + +REPO = Path(__file__).resolve().parents[2] +os.chdir(REPO) # the training module inserts repo-relative third_party paths + +import numpy as np # noqa: E402 +import torch # noqa: E402 +import torch.nn.functional as F # noqa: E402 +from PIL import Image # noqa: E402 + +TRAIN_SCRIPT = REPO / "scripts/parallel_sharpa_forward_checks/run_giga_sharpa_train_job.py" +PREP_DIR = REPO / "assets/giga_sharpa_train_job/prep_eef_full_v1" +VAE_DIR = REPO / "assets/video_backbone_ckpt/Wan2.2-VAE-Diffusers/vae" + + +def load_train_module(): + spec = importlib.util.spec_from_file_location("gwp_train", TRAIN_SCRIPT) + T = importlib.util.module_from_spec(spec) + spec.loader.exec_module(T) + # main() sets these for --action-repr eef --views 2 --view-concat t + T.ACTION_DIM = T.EEF_DIM + T.TRANSFORMER_CFG.update(in_action_channels=T.EEF_DIM, out_action_channels=T.EEF_DIM) + T.VIEWS, T.VIEW_CONCAT = 2, "t" + return T + + +class GWPSharpaPolicy: + def __init__(self, ckpt: str, device: str = "cuda", num_steps: int = 10, prep_dir: Path = PREP_DIR): + self.T = T = load_train_module() + from diffusers.models import AutoencoderKLWan + from world_action_model.models import CasualWorldActionTransformer_MoT + + self.device = torch.device(device) + self.num_steps = int(num_steps) + t0 = time.time() + model = CasualWorldActionTransformer_MoT(**T.TRANSFORMER_CFG) + sd = torch.load(ckpt, map_location="cpu", mmap=True, weights_only=False) + model.load_state_dict(sd, strict=True) + del sd + self.model = model.to(self.device, dtype=torch.bfloat16).eval() + self.vae = AutoencoderKLWan.from_pretrained(str(VAE_DIR)).to(self.device, dtype=torch.float32).eval() + self.vae.requires_grad_(False) + norm = json.load(open(prep_dir / "norm_stats.json"))["norm_stats"] + s, a = norm["observation.state"], norm["action"] + self.s_lo, self.s_hi = np.asarray(s["q01"], np.float32), np.asarray(s["q99"], np.float32) + self.a_lo, self.a_hi = np.asarray(a["q01"], np.float32), np.asarray(a["q99"], np.float32) + t5 = torch.load(prep_dir / "t5_task_embeddings.pt", map_location="cpu", weights_only=False) + self.prompt_embeds = {k: v.float() for k, v in t5["by_text"].items()} + # training's view composition, bound to a val-mode (center-crop) stand-in for the dataset object + self._views = types.SimpleNamespace(train=False) + self._views._process_images = types.MethodType(T.SharpaGigaDataset._process_images, self._views) + self._views._compose_views = types.MethodType(T.SharpaGigaDataset._compose_views, self._views) + self.ckpt = ckpt + print(f"[policy] loaded {ckpt} in {time.time() - t0:.1f}s; steps={self.num_steps}", flush=True) + + def preprocess(self, images: dict, state: np.ndarray, task: str) -> dict: + T = self.T + cams = T.EEF_CAMERAS[: T.VIEWS] + video = {c: [Image.fromarray(np.asarray(images[c.split(".")[-1]], np.uint8))] * len(T.IMAGE_FRAME_OFFSETS) + for c in cams} + imgs = self._views._compose_views(video) # [5,3,384,320] in [-1,1]; only frame 0 conditions the action + st = T.SharpaGigaDataset._quantile_norm(np.asarray(state, np.float32)[None], self.s_lo, self.s_hi) + if task not in self.prompt_embeds: + raise KeyError(f"task {task!r} has no training prompt embedding") + p = self.prompt_embeds[task][: T.MAX_PROMPT_LEN] + p = F.pad(p, (0, 0, 0, T.MAX_PROMPT_LEN - p.shape[0]), value=0) + return {"images": imgs[None], "state": torch.from_numpy(st.astype(np.float32))[None], + "prompt_embeds": p[None]} + + @torch.no_grad() + def predict(self, images: dict, state: np.ndarray, task: str, seed: int = 0, num_steps: int | None = None): + raw = self.preprocess(images, state, task) + a = self.T.sample_actions(self.model, self.vae, raw, self.device, num_steps=num_steps or self.num_steps, + seed=int(seed))[0].cpu().numpy() + return (a + 1.0) / 2.0 * np.maximum(self.a_hi - self.a_lo, 1e-8) + self.a_lo + + def inference(self, data: dict) -> dict: + t0 = time.perf_counter() + action = self.predict(data["images"], data["state"], data["task"], data.get("seed", 0), data.get("num_steps")) + torch.cuda.synchronize(self.device) + return {"action": action.astype(np.float32), "timing": {"server_ms": (time.perf_counter() - t0) * 1e3}, + "ckpt": self.ckpt} + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--ckpt", required=True) + p.add_argument("--port", type=int, default=11411) + p.add_argument("--num-steps", type=int, default=10) + p.add_argument("--prep-dir", type=Path, default=PREP_DIR, help="training prep dir (norm_stats.json, t5_task_embeddings.pt)") + a = p.parse_args() + sys.path.insert(0, str(REPO / "third_party/giga-world-policy/third_party/giga-models")) + from giga_models.sockets.server import RobotInferenceServer + + policy = GWPSharpaPolicy(a.ckpt, num_steps=a.num_steps, prep_dir=a.prep_dir) + RobotInferenceServer.start_server(policy, a.port) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/sharpa_webxr/postprocess.py b/benchmarks/sharpa_webxr/postprocess.py new file mode 100644 index 00000000..7388e0d8 --- /dev/null +++ b/benchmarks/sharpa_webxr/postprocess.py @@ -0,0 +1,274 @@ +"""Offline pass over a saved rollout (trajectory.npz qpos): rollout video + automatic stage signals. + +Video: ego_view | chest_view side by side, 320x240 each, 10 fps (every other 20 Hz step) -> video.mp4. +Signals (generic, every free-floating or articulated scene object; thresholds provisional): + reach : any hand geom within 3 cm of the object (mj_geomDistance) + grasp : hand-object contact sustained >= 0.5 s while the object is lifted >= 2 cm or carried >= 3 cm + move : object displaced >= 5 cm from its initial position, or an object hinge/slide joint moved >= 20 deg / 3 cm + fallen : object below the table top by > 10 cm +Everything is computed on the first limits.limit(scene) steps (min(1.5 x p90, 20 s)), so rollouts recorded with a +longer limit score exactly like capped ones (override with --max-steps). +Judge frames (--judge-frames K, or --frames-only): K evenly spaced full-resolution frames per camera, saved as +judge_frames/t_.jpg for score.py. +For close_laptop / box_lid scenes the benchmarks/sharpa/verifiers.py rule check also runs (success_any: passed at +any time; success_final: passed at the end), same as RoboTwin's "done on first success" semantics. +""" +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +import numpy as np + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +sys.path.insert(0, str(HERE.parents[1])) +from env import IMAGE_HW, SharpaWebXREnv # noqa: E402 +from limits import limit # noqa: E402 + +import mujoco # noqa: E402 + +SCENES_ROOT = HERE.parents[1] / "data/webxr_scenes" +DT = 0.05 # 20 Hz rows +REACH_M, LIFT_M, CARRY_M, MOVE_M, HOLD_S = 0.03, 0.02, 0.03, 0.05, 0.5 +JOINT_MOVE_RAD, JOINT_MOVE_M = np.deg2rad(20), 0.03 + + +def body_root_tree(m): + """body id -> top-level child-of-world ancestor id.""" + root = np.zeros(m.nbody, int) + for b in range(m.nbody): + a = b + while m.body_parentid[a] != 0: + a = m.body_parentid[a] + root[b] = a + return root + + +class Analyzer: + def __init__(self, env: SharpaWebXREnv): + self.env = env + m = self.m = env.sim.model + self.d = env.sim.data + name = lambda t, i: mujoco.mj_id2name(m, t, i) or "" # noqa: E731 + robot_bodies = {int(m.jnt_bodyid[int(m.actuator_trnid[a, 0])]) for a in env.robot_act} + root = body_root_tree(m) + robot_roots = {root[b] for b in robot_bodies} + # hand geoms: geoms of bodies named left_*/right_* belonging to hand chains (not arm links) + self.hand_geoms = np.array([g for g in range(m.ngeom) if root[m.geom_bodyid[g]] in robot_roots + and any(k in name(mujoco.mjtObj.mjOBJ_BODY, m.geom_bodyid[g]) + for k in ("thumb", "index", "middle", "ring", "pinky", "palm", "hand")) + and m.geom_contype[g] | m.geom_conaffinity[g]]) + # scene objects: world children with a free joint, not robot + self.objects = {} + for b in range(1, m.nbody): + if m.body_parentid[b] != 0 and root[b] != b: + continue + if root[b] in robot_roots or m.body_jntnum[b] == 0 or int(m.jnt_type[m.body_jntadr[b]]) != int(mujoco.mjtJoint.mjJNT_FREE): + continue + sub = [x for x in range(m.nbody) if root[x] == b] + geoms = np.array([g for g in range(m.ngeom) if m.geom_bodyid[g] in sub and m.geom_contype[g] | m.geom_conaffinity[g]]) + # numpy int32 vs pybind enum: `in (enum, ...)` is always False, so compare as ints + joints = [j for j in range(m.njnt) if m.jnt_bodyid[j] in sub and int(m.jnt_type[j]) in + (int(mujoco.mjtJoint.mjJNT_HINGE), int(mujoco.mjtJoint.mjJNT_SLIDE))] + self.objects[name(mujoco.mjtObj.mjOBJ_BODY, b)] = {"body": b, "sub": set(sub), "geoms": geoms, "joints": joints} + self.root = root + self.geom_obj = {} + for o, v in self.objects.items(): + for g in v["geoms"]: + self.geom_obj[int(g)] = o + self.hand_set = set(int(g) for g in self.hand_geoms) + tid = mujoco.mj_name2id(m, mujoco.mjtObj.mjOBJ_GEOM, "world_table") + self.table_z = float(self.d.geom_xpos[tid][2] + m.geom_size[tid][2]) if tid >= 0 else None + + def frame(self, qpos): + d, m = self.d, self.m + d.qpos[:] = qpos + d.qvel[:] = 0 + mujoco.mj_forward(m, d) + contact = {o: False for o in self.objects} + pairs = set() + for i in range(d.ncon): + c = d.contact[i] + g1, g2 = int(c.geom1), int(c.geom2) + for a, b in ((g1, g2), (g2, g1)): + if a in self.hand_set and b in self.geom_obj: + contact[self.geom_obj[b]] = True + b1 = mujoco.mj_id2name(m, mujoco.mjtObj.mjOBJ_BODY, self.root[m.geom_bodyid[g1]]) or "" + b2 = mujoco.mj_id2name(m, mujoco.mjtObj.mjOBJ_BODY, self.root[m.geom_bodyid[g2]]) or "" + n1 = mujoco.mj_id2name(m, mujoco.mjtObj.mjOBJ_BODY, m.geom_bodyid[g1]) or "" + n2 = mujoco.mj_id2name(m, mujoco.mjtObj.mjOBJ_BODY, m.geom_bodyid[g2]) or "" + pairs.add((n1, n2)); pairs.add((b1, b2)) + out = {} + for o, v in self.objects.items(): + dist = np.inf + if not contact[o]: + fromto = np.zeros(6) + for g in v["geoms"][:64]: + for h in self.hand_geoms: + dist = min(dist, mujoco.mj_geomDistance(m, d, int(h), int(g), REACH_M * 2, fromto)) + if dist < REACH_M: + break + if dist < REACH_M: + break + else: + dist = 0.0 + out[o] = {"pos": d.xpos[v["body"]].copy(), "quat": d.xquat[v["body"]].copy(), + "joints": np.array([d.qpos[m.jnt_qposadr[j]] for j in v["joints"]]), + "contact": contact[o], "dist": float(dist)} + wrist = np.stack([self.env.sim.wrist_pose(s)[0] for s in ("left", "right")]) + return out, pairs, wrist + + +def snapshot(m, d, t, pairs): + from benchmarks.sharpa.verifiers import Snapshot + + pos = {mujoco.mj_id2name(m, mujoco.mjtObj.mjOBJ_BODY, b): d.xpos[b].copy() for b in range(m.nbody)} + quat = {mujoco.mj_id2name(m, mujoco.mjtObj.mjOBJ_BODY, b): d.xquat[b].copy() for b in range(m.nbody)} + joints = {mujoco.mj_id2name(m, mujoco.mjtObj.mjOBJ_JOINT, j): float(d.qpos[m.jnt_qposadr[j]]) for j in range(m.njnt)} + return Snapshot(t, pos, quat, joints, pairs) + + +VIEWS = ("ego_view", "chest_view") + + +def load_qpos(rollout_dir: Path, max_steps: int | None = None): + meta = json.loads((rollout_dir / "rollout.json").read_text()) + n = max_steps if max_steps is not None else limit(meta["scene_id"]) + return meta, np.load(rollout_dir / "trajectory.npz")["qpos"][: n + 1] # row 0 = reset state + + +def judge_frames(rollout_dir: Path, env: SharpaWebXREnv, k: int = 24, max_steps: int | None = None) -> int: + """K evenly spaced frames per camera at the native 480x640 -> judge_frames/t_.jpg.""" + import imageio.v2 as imageio + + _, qpos = load_qpos(rollout_dir, max_steps) + out = rollout_dir / "judge_frames" + out.mkdir(exist_ok=True) + # COMPLETE is written last: a render killed mid-write (e.g. a watcher restart) is redone on the next call + # instead of being judged on the frames it got to. Verdicts on the old frames are dropped with them. + (out / "COMPLETE").unlink(missing_ok=True) + for old in out.glob("*.jpg"): + old.unlink() + for verdict in rollout_dir.glob("judge_*_r2.json"): + verdict.unlink() + for t in sorted(set(np.linspace(0, len(qpos) - 1, k).round().astype(int).tolist())): + env.sim.data.qpos[:] = qpos[t] + mujoco.mj_forward(env.sim.model, env.sim.data) + for v in VIEWS: + imageio.imwrite(out / f"t{t * DT:07.2f}_{v}.jpg", env.render(v, IMAGE_HW), quality=90) + (out / "COMPLETE").touch() + return len(qpos) - 1 + + +def analyze(rollout_dir: Path, env: SharpaWebXREnv, video: bool = True, keyframes: int = 16, + max_steps: int | None = None) -> dict: + meta, qpos = load_qpos(rollout_dir, max_steps) + an = Analyzer(env) + from benchmarks.sharpa.verifiers import SCENES, Verifier + + vtask = {v: k for k, v in SCENES.items()}.get(meta["scene_id"]) + ver, ver_any = (Verifier(vtask), False) if vtask else (None, None) + frames, rows, keys = [], [], {} + key_idx = set(np.linspace(0, len(qpos) - 1, keyframes).round().astype(int).tolist()) if keyframes else set() + for t in range(len(qpos)): + o, pairs, wrist = an.frame(qpos[t]) + rows.append(o) + if ver is not None: + snap = snapshot(an.m, an.d, t * DT, pairs) + if t == 0: + ver.reset(snap) + else: + r = ver.update(snap) + ver_any = ver_any or r["success"] + if (video and t % 2 == 0) or t in key_idx: + img = np.concatenate([env.render(v, (240, 320)) for v in ("ego_view", "chest_view")], axis=1) + if video and t % 2 == 0: + frames.append(img) + if t in key_idx: + keys[t] = img + sig = {} + for name in an.objects: + P = np.stack([r[name]["pos"] for r in rows]) + C = np.array([r[name]["contact"] for r in rows]) + D = np.array([r[name]["dist"] for r in rows]) + J = np.stack([r[name]["joints"] for r in rows]) if len(an.objects[name]["joints"]) else None + disp = np.linalg.norm(P - P[0], axis=1) + lift = P[:, 2] - P[0, 2] + hold_n = int(round(HOLD_S / DT)) + run, grasp = 0, False + for t in range(len(P)): + run = run + 1 if C[t] else 0 + if run >= hold_n and (lift[t] >= LIFT_M or disp[t] >= CARRY_M): + grasp = True + jmove = False + if J is not None: + types = [int(an.m.jnt_type[j]) for j in an.objects[name]["joints"]] + dj = np.abs(J - J[0]).max(axis=0) + jmove = any(dj[i] >= (JOINT_MOVE_RAD if ty == int(mujoco.mjtJoint.mjJNT_HINGE) else JOINT_MOVE_M) + for i, ty in enumerate(types)) + sig[name] = {"reach": bool((D < REACH_M).any()), "contact_s": float(C.sum() * DT), "grasp": grasp, + "move": bool(disp.max() >= MOVE_M or jmove), "max_disp_m": float(disp.max()), + "max_lift_m": float(lift.max()), "final_disp_m": float(disp[-1]), + "fallen": bool(an.table_z is not None and P[:, 2].min() < an.table_z - 0.10), + "max_joint_change": (np.abs(J - J[0]).max(axis=0).tolist() if J is not None else [])} + any_ = lambda k: any(v[k] for v in sig.values()) # noqa: E731 + out = {"scene_id": meta["scene_id"], "seed": meta["seed"], "objects": sig, + "stage": {"reach": any_("reach"), "grasp": any_("grasp"), "move": any_("move")}, + "fallen_any": any_("fallen"), "ik_converged_frac": meta.get("ik_converged_frac"), + "analyzed_steps": len(qpos) - 1, "recorded_steps": meta["steps"]} + if ver is not None: + fin = ver.finalize() + out["verifier"] = {"task": vtask, "success_any": bool(ver_any), "success_final": bool(fin["success"]), + "failed_checks_final": fin["failed_checks"], "metrics_final": fin["metrics"]} + (rollout_dir / "signals.json").write_text(json.dumps(out, indent=1)) + if keys: + import imageio.v2 as imageio + + kd = rollout_dir / "keyframes" + kd.mkdir(exist_ok=True) + for old in kd.glob("*.jpg"): + old.unlink() + for t, img in keys.items(): + imageio.imwrite(kd / f"t{t * DT:07.2f}.jpg", img, quality=88) + if video: + import imageio.v2 as imageio + + imageio.mimwrite(rollout_dir / "video.mp4", frames, fps=10, quality=7, macro_block_size=8) + return out + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("dirs", nargs="+", type=Path, help="rollout dirs (…//seed_k)") + p.add_argument("--no-video", action="store_true") + p.add_argument("--keyframes", type=int, default=16, help="evenly spaced JPEG frames for the judge (0 = off)") + p.add_argument("--force", action="store_true") + p.add_argument("--max-steps", type=int, default=None, help="analyze the first N steps (default: limits.limit)") + p.add_argument("--judge-frames", type=int, default=0, help="also write K full-res frames per camera for score.py") + p.add_argument("--frames-only", action="store_true", help="only (re)write judge frames (implies --judge-frames 24)") + a = p.parse_args() + if a.frames_only and not a.judge_frames: + a.judge_frames = 24 + env_cache = {} + for d in a.dirs: + need_frames = a.judge_frames and (a.force or not (d / "judge_frames" / "COMPLETE").exists()) + need_signals = not a.frames_only and (a.force or not (d / "signals.json").exists()) + if not (need_frames or need_signals): + continue + scene = json.loads((d / "rollout.json").read_text())["scene_id"] + if scene not in env_cache: + env_cache.clear() + env_cache[scene] = SharpaWebXREnv(SCENES_ROOT / scene) + if need_signals: + r = analyze(d, env_cache[scene], video=not a.no_video, keyframes=a.keyframes, max_steps=a.max_steps) + print(d, r["stage"], r.get("verifier", {}).get("success_any"), f"steps {r['analyzed_steps']}", flush=True) + if need_frames: + n = judge_frames(d, env_cache[scene], a.judge_frames, a.max_steps) + print(d, f"judge frames over {n} steps", flush=True) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/sharpa_webxr/pull_episodes.py b/benchmarks/sharpa_webxr/pull_episodes.py new file mode 100644 index 00000000..9415733e --- /dev/null +++ b/benchmarks/sharpa_webxr/pull_episodes.py @@ -0,0 +1,93 @@ +"""Pull human-labeled teleop episodes (success / recover / failure) into the rollout format for verifier calibration. + +Source: gs://foundational-research/webxr-teleop/live/indexes/recordings.json (one entry per episode: outcome, +scene_id, scene_commit, trajectory_key, ...). Labels: success = done; recover = done after a retry (suboptimal); +failure = not done. 20 Hz rows = initial_state qpos + states/qpos[2::3], as in make_gt_rollouts.py. + +Output: //_/{trajectory.npz, rollout.json}; scene.xml is saved too when the +episode's scene revision differs from data/webxr_scenes/ (replay needs the recorded scene then). +usage: pull_episodes.py --out data/webxr_episodes [--scenes ...] [--outcomes success recover failure] [--workers 16] +""" +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import tempfile +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +import h5py +import numpy as np + +LIVE = "gs://foundational-research/webxr-teleop/live" +ROOT = Path(__file__).resolve().parents[2] +TASKS = json.loads((Path(__file__).resolve().parent / "tasks.json").read_text()) +SCENES_ROOT = ROOT / "data/webxr_scenes" + + +def local_revision(scene_id: str) -> str | None: + st = SCENES_ROOT / scene_id / ".revision.json" + return json.loads(st.read_text()).get("revision") if st.exists() else None + + +def pull_one(e: dict, out: Path) -> str: + sid = e["scene_id"] + d = out / sid / f"{e['outcome']}_{e['episode_uuid'][:8]}" + if (d / "rollout.json").exists(): + return "skip" + with tempfile.TemporaryDirectory() as tmp: + subprocess.run(["gsutil", "-q", "cp", f"{LIVE}/{e['trajectory_key']}", tmp], check=True) + with h5py.File(Path(tmp) / "trajectory.hdf5") as f: + g = f["data/demo_0"] + q = np.concatenate([g["initial_state/qpos"][:][None], g["states/qpos"][2::3]]) + xml = bytes(g["source_files/scene.xml"][:]).decode() if "source_files/scene.xml" in g else None + rev_match = e.get("scene_commit") == local_revision(sid) + d.mkdir(parents=True, exist_ok=True) + np.savez_compressed(d / "trajectory.npz", qpos=q) + if not rev_match and xml: + (d / "scene.xml").write_text(xml) + meta = {"scene_id": sid, "seed": d.name, "gt": True, "outcome": e["outcome"], "completed": e["outcome"] != "failure", + "episode_uuid": e["episode_uuid"], "user_id": e.get("user_id"), "upload_date": e.get("upload_date"), + "scene_commit": e.get("scene_commit"), "revision_match": rev_match, "source_num_steps": int(e["num_steps"]), + "task": TASKS[sid]["task"], "steps": len(q) - 1} + (d / "rollout.json").write_text(json.dumps(meta, indent=1)) + return "ok" + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--out", type=Path, default=ROOT / "data/webxr_episodes") + p.add_argument("--scenes", nargs="*", default=None, help="default: every scene in tasks.json") + p.add_argument("--outcomes", nargs="*", default=["success", "recover", "failure", "suboptimal"]) + p.add_argument("--workers", type=int, default=16) + p.add_argument("--index", type=Path, default=None, help="local copy of recordings.json (default: fetch)") + a = p.parse_args() + if a.index: + idx = json.loads(a.index.read_text())["entries"] + else: + idx = json.loads(subprocess.run(["gsutil", "-q", "cat", f"{LIVE}/indexes/recordings.json"], check=True, + capture_output=True).stdout)["entries"] + scenes = set(a.scenes or TASKS) + eps = sorted((e for e in idx.values() if e["scene_id"] in scenes and e["outcome"] in a.outcomes), + key=lambda e: (e["scene_id"], e["outcome"], e["episode_uuid"])) + a.out.mkdir(parents=True, exist_ok=True) + (a.out / "recordings_index.json").write_text(json.dumps({"entries": {e["episode_uuid"]: e for e in eps}}, indent=0)) + print(f"{len(eps)} episodes, {len(scenes)} scenes -> {a.out}", flush=True) + done = {"ok": 0, "skip": 0, "err": 0} + with ThreadPoolExecutor(a.workers) as ex: + def run(e): + try: + return e, pull_one(e, a.out) + except Exception as ex_: # noqa: BLE001 one bad episode must not stop the pull + return e, f"err {type(ex_).__name__}: {ex_}" + for i, (e, r) in enumerate(ex.map(run, eps), 1): + done["err" if r.startswith("err") else r] += 1 + if r.startswith("err") or i % 100 == 0: + print(f"[{i}/{len(eps)}] {e['scene_id'][:30]} {e['outcome']} {r}", flush=True) + print("DONE", done, flush=True) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/sharpa_webxr/render_video.py b/benchmarks/sharpa_webxr/render_video.py new file mode 100644 index 00000000..e540b6a1 --- /dev/null +++ b/benchmarks/sharpa_webxr/render_video.py @@ -0,0 +1,33 @@ +"""Video only (no signals): ego | chest at 320x240 each, 10 fps, from trajectory.npz -> video.mp4. + +Covers the first limits.limit(scene) steps (min(1.5 x p90, 20 s)), the same span the signals and judge score. +""" +import json +import os +import sys +from pathlib import Path + +import imageio.v2 as imageio +import mujoco +import numpy as np + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from env import SharpaWebXREnv # noqa: E402 +from limits import limit # noqa: E402 + +SCENES = Path(__file__).resolve().parents[2] / "data/webxr_scenes" +for d in map(Path, sys.argv[1:]): + if (d / "video.mp4").exists(): + continue + meta = json.loads((d / "rollout.json").read_text()) + env = SharpaWebXREnv(SCENES / meta["scene_id"]) + q = np.load(d / "trajectory.npz")["qpos"][: limit(meta["scene_id"]) + 1] + frames = [] + for t in range(0, len(q), 2): + env.sim.data.qpos[:] = q[t] + mujoco.mj_forward(env.sim.model, env.sim.data) + frames.append(np.concatenate([env.render(v, (240, 320)) for v in ("ego_view", "chest_view")], axis=1)) + tmp = d / f"video.{os.getpid()}.tmp.mp4" # atomic: concurrent renderers of the same dir never leave a torn file + imageio.mimwrite(tmp, frames, fps=10, quality=7, macro_block_size=8) + os.replace(tmp, d / "video.mp4") + print("wrote", d / "video.mp4", len(frames), flush=True) diff --git a/benchmarks/sharpa_webxr/review_page.css b/benchmarks/sharpa_webxr/review_page.css new file mode 100644 index 00000000..9e35fa91 --- /dev/null +++ b/benchmarks/sharpa_webxr/review_page.css @@ -0,0 +1,41 @@ +/* layout: header with run facts, then one row per task: policy seeds beside the teleop demo */ +:root{ + --bg:#f3f4f2; --panel:#ffffff; --fg:#1d2321; --muted:#5d6763; --line:#d9dedb; --accent:#1f6f5c; + --ok:#1d7a46; --ok-bg:#e2f3e8; --no:#a23b2a; --no-bg:#f8e4df; --na:#6b7470; --na-bg:#eceeed; + --display:"IBM Plex Sans Condensed","Arial Narrow",sans-serif; --body:"IBM Plex Sans",system-ui,sans-serif; --mono:"IBM Plex Mono",ui-monospace,monospace; +} +@media (prefers-color-scheme: dark){:root:not([data-theme="light"]){ + --bg:#151918; --panel:#1d2321; --fg:#e4e9e6; --muted:#9aa5a0; --line:#323a37; --accent:#5cc2a4; + --ok:#7fd6a0; --ok-bg:#1f3a2a; --no:#f0a090; --no-bg:#43251f; --na:#a3aca8; --na-bg:#2a302e; color-scheme:dark}} +:root[data-theme="dark"]{ + --bg:#151918; --panel:#1d2321; --fg:#e4e9e6; --muted:#9aa5a0; --line:#323a37; --accent:#5cc2a4; + --ok:#7fd6a0; --ok-bg:#1f3a2a; --no:#f0a090; --no-bg:#43251f; --na:#a3aca8; --na-bg:#2a302e; color-scheme:dark} +body{background:var(--bg);color:var(--fg);font-family:var(--body);font-size:15px;line-height:1.5} +.wrap{max-width:1280px;margin:0 auto;padding-inline:20px;padding-block:28px 48px;display:flex;flex-direction:column;gap:28px} +h1{font-family:var(--display);font-weight:600;font-size:30px;margin:0;text-wrap:balance} +h2{font-family:var(--display);font-weight:600;font-size:20px;margin:0} +.lede{color:var(--muted);max-width:72ch;margin:6px 0 0} +.facts{display:flex;flex-wrap:wrap;gap:8px 22px;font-family:var(--mono);font-size:12.5px;color:var(--muted)} +.facts b{color:var(--fg);font-weight:500} +.note{border-left:3px solid var(--accent);padding:8px 14px;background:var(--panel);max-width:90ch;font-size:14px} +.task{display:flex;flex-direction:column;gap:12px} +.task .prompt{font-family:var(--mono);font-size:13px;color:var(--muted)} +.row{display:grid;grid-template-columns:repeat(auto-fill,minmax(380px,1fr));gap:16px} +.card{background:var(--panel);border:1px solid var(--line);border-radius:6px;overflow:hidden;display:flex;flex-direction:column;min-width:0} +.card.gt{border-style:dashed} +video{width:100%;display:block;background:#000;aspect-ratio:640/240} +.meta{padding:10px 12px 12px;display:flex;flex-direction:column;gap:8px} +.top{display:flex;justify-content:space-between;align-items:baseline;gap:8px} +.label{font-family:var(--display);font-weight:600;font-size:15px} +.kind{font-family:var(--mono);font-size:11px;letter-spacing:.06em;text-transform:uppercase;color:var(--muted)} +.nums{font-family:var(--mono);font-size:12px;color:var(--muted);font-variant-numeric:tabular-nums} +.chips{display:flex;flex-wrap:wrap;gap:6px} +.chip{font-family:var(--mono);font-size:11.5px;padding:2px 8px;border-radius:999px} +.chip.ok{background:var(--ok-bg);color:var(--ok)} .chip.no{background:var(--no-bg);color:var(--no)} .chip.na{background:var(--na-bg);color:var(--na)} +.views{font-family:var(--mono);font-size:11px;color:var(--muted);display:flex;justify-content:space-between;padding:4px 12px 0} + +.tablewrap{overflow-x:auto} table{border-collapse:collapse;font-family:var(--mono);font-size:13px;font-variant-numeric:tabular-nums} +th,td{padding:6px 12px;border-bottom:1px solid var(--line);text-align:right} th[scope=row],thead th:first-child{text-align:left;font-family:var(--body)} +thead th{color:var(--muted);font-weight:500} .head{display:flex;flex-wrap:wrap;align-items:baseline;gap:4px 14px} .sid{font-family:var(--mono);font-size:12px;color:var(--muted);overflow-wrap:anywhere} +figure{margin:0} figcaption{padding:10px 12px 12px;display:flex;flex-direction:column;gap:8px} +.row{grid-template-columns:repeat(auto-fill,minmax(min(380px,100%),1fr))} diff --git a/benchmarks/sharpa_webxr/rollout.py b/benchmarks/sharpa_webxr/rollout.py new file mode 100644 index 00000000..618c274a --- /dev/null +++ b/benchmarks/sharpa_webxr/rollout.py @@ -0,0 +1,144 @@ +"""Closed-loop rollout of any policy server (policies/_server.py) in the sim; runs in .venv_webxr. + +Sim side of the policies/common.py contract, the same for every model: +- the server's "meta" says how it is deployed: ``execute_steps`` (actions run before replanning, matched to the + model's upstream deploy; --execute-steps overrides) and ``obs_offsets`` (rows <= 0 it needs observations at, e.g. + LDA-1B's t-5; rows before the episode start repeat the first observation); +- the env (env.py, action_space "eef") is asked to observe only at those rows, and executes the returned absolute + wrist poses + hand joints (IK inside the env, seeded from the previous solve); +- loop: observe -> request a chunk -> execute its first execute_steps -> re-observe; the queue is cleared on every + new chunk, no ensembling. Episode length: limits.limit(scene) = min(1.5 x p90, 20 s). +Saves the full qpos trajectory; signals, judge frames and videos are computed offline (postprocess.py, render_video.py). + +usage: rollout.py --port --scene --seeds 0-9 --out [--execute-steps N] [--max-steps N] +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +import time +from pathlib import Path + +import numpy as np + +HERE = Path(__file__).resolve().parent +REPO = HERE.parents[1] +sys.path.insert(0, str(HERE)) +from env import IMAGE_HW, ObsConfig, SharpaWebXREnv # noqa: E402 +from limits import limit # noqa: E402 +from policies.common import canon_to_dict, dict_to_canon # noqa: E402 + +TASKS = json.loads((HERE / "tasks.json").read_text()) +SCENES_ROOT = REPO / "data/webxr_scenes" +SIDES = ("left", "right") + + +def seed_for(scene_id: str, seed: int) -> int: + """Reset / policy-noise seed for (scene, seed): identical across checkpoints and models.""" + return int.from_bytes(hashlib.sha256(f"{scene_id}/{seed}".encode()).digest()[:4], "little") + + +class PolicyClient: + def __init__(self, port: int, host: str = "localhost", timeout_ms: int = 600000): + sys.path.insert(0, str(REPO / "third_party/giga-world-policy/third_party/giga-models/giga_models/sockets")) + from client import RobotInferenceClient # upstream ZeroMQ client (file import: no giga_models package deps) + + self.c = RobotInferenceClient(host=host, port=port, timeout_ms=timeout_ms) + assert self.c.ping(), f"policy server on {host}:{port} not answering" + self.meta = self.c.call_endpoint("meta", requires_input=False) + + def __call__(self, req: dict) -> dict: + return self.c.inference(req) + + +def make_env(meta: dict, scene_id: str) -> SharpaWebXREnv: + cams = meta.get("cameras") or {"ego_view": IMAGE_HW, "chest_view": IMAGE_HW} + return SharpaWebXREnv(SCENES_ROOT / scene_id, obs=ObsConfig(cameras=cams, state=("eef",)), + action_space="eef", eef_frame="ego_view") + + +def _pack(obs: dict) -> dict: + return {"images": obs["images"], "state": canon_to_dict(obs["eef"]["canonical"])} + + +def run_episode(env: SharpaWebXREnv, policy: PolicyClient, scene_id: str, seed: int, out_dir: Path, + execute_steps: int, max_steps: int | None = None) -> dict: + task = TASKS[scene_id]["task"] + max_steps = max_steps or limit(scene_id) + offsets = sorted(int(o) for o in policy.meta["obs_offsets"]) + t0 = time.time() + record = {0: _pack(env.reset(seed_for(scene_id, seed)))} # row -> observation the server may still ask for + qpos, targets, ik_pe, ik_re, ik_ok, requests = [env.sim.data.qpos.copy()], [], [], [], [], [] + prev_cmd, executed, resp = None, None, {} + step, chunk_i = 0, 0 + while step < max_steps: + tr = time.perf_counter() + resp = policy({"obs": {o: record[max(step + o, 0)] for o in offsets}, "task": task, + "seed": seed_for(scene_id, seed) + chunk_i, "reset": chunk_i == 0, "prev_cmd": prev_cmd, + "executed": executed}) + requests.append({"step": step, "rpc_ms": (time.perf_counter() - tr) * 1e3, **resp.get("timing", {})}) + cmd = dict_to_canon({k: np.asarray(resp[k])[:execute_steps] for k in ("pos", "rot", "hand")}) + need = {max(step + len(cmd.pos) + o, 0) for o in offsets} # rows the next request will read + record = {k: v for k, v in record.items() if k in need} + step0 = step + for t in range(len(cmd.pos)): + if step >= max_steps: + break + a = {} + for k, s in enumerate(SIDES): + T = np.eye(4) + T[:3, :3], T[:3, 3] = cmd.rot[t, k], cmd.pos[t, k] + a[s] = {"pose": T, "hand": cmd.hand[t, k]} + obs, info = env.step(a, observe=step + 1 in need) + if obs is not None: + record[step + 1] = _pack(obs) + qpos.append(env.sim.data.qpos.copy()) + targets.append(env.last_target.copy()) + ik_pe.append(info["ik_pos_err"]); ik_re.append(info["ik_rot_err"]); ik_ok.append(info["ik_converged"]) + step += 1 + executed = step - step0 + prev_cmd = canon_to_dict(cmd[executed - 1:executed]) # last commanded pose (LDA anchor) + chunk_i += 1 + out_dir.mkdir(parents=True, exist_ok=True) + np.savez_compressed(out_dir / "trajectory.npz", qpos=np.asarray(qpos), targets=np.asarray(targets, np.float32), + ik_pos_err=np.asarray(ik_pe, np.float32), ik_rot_err=np.asarray(ik_re, np.float32), + ik_converged=np.asarray(ik_ok), joint_names=np.asarray(env.joint_names)) + meta = {"scene_id": scene_id, "task": task, "seed": seed, "reset_seed": seed_for(scene_id, seed), + "randomization": env.randomization, "steps": step, "max_steps": max_steps, "execute_steps": execute_steps, + "chunks": chunk_i, "wall_s": time.time() - t0, "ckpt": resp.get("ckpt"), "policy_meta": policy.meta, + "requests": requests, "ik_converged_frac": float(np.mean(ik_ok)) if ik_ok else None, + "ik_pos_err_p95_mm": float(np.quantile(np.asarray(ik_pe), 0.95) * 1e3) if ik_pe else None} + (out_dir / "rollout.json").write_text(json.dumps(meta, indent=1, default=str)) + return meta + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--port", type=int, required=True) + p.add_argument("--scene", required=True) + p.add_argument("--seeds", default="0-9", help="e.g. 0-9 or 0,3,5") + p.add_argument("--out", type=Path, required=True) + p.add_argument("--execute-steps", type=int, default=None, help="default: the server's meta (upstream-matched)") + p.add_argument("--max-steps", type=int, default=None, help="default: limits.limit(scene)") + a = p.parse_args() + seeds = (list(range(int(a.seeds.split("-")[0]), int(a.seeds.split("-")[1]) + 1)) if "-" in a.seeds + else [int(s) for s in a.seeds.split(",")]) + policy = PolicyClient(a.port) + execute = a.execute_steps or int(policy.meta["execute_steps"]) + print(f"[rollout] {policy.meta.get('model')} execute_steps={execute} obs_offsets={policy.meta['obs_offsets']}", + flush=True) + env = make_env(policy.meta, a.scene) + for s in seeds: + d = a.out / a.scene / f"seed_{s}" + if (d / "rollout.json").exists(): + print(f"[skip] {d}", flush=True) + continue + m = run_episode(env, policy, a.scene, s, d, execute, a.max_steps) + print(f"[done] {a.scene} seed {s}: {m['steps']} steps, {m['chunks']} chunks, {m['wall_s']:.0f}s, " + f"ik_conv {m['ik_converged_frac']:.3f}", flush=True) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/sharpa_webxr/run_suite.sh b/benchmarks/sharpa_webxr/run_suite.sh new file mode 100755 index 00000000..a3060381 --- /dev/null +++ b/benchmarks/sharpa_webxr/run_suite.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# Closed-loop sweep of one checkpoint: 58 training scenes x seeds, longest scenes first. +# usage: RUN= [EXECUTE_STEPS=N] run_suite.sh [seeds_per_job] +# Each job: rollout.py (closed loop vs the policy server on ) then postprocess.py (stage signals, rule +# verifier, 16 judge keyframes; no full video). Resumable: finished seeds (rollout.json / signals.json) are skipped. +set -u +cd "$(dirname "$(readlink -f "$0")")/../.." # repo root +TAG=$1; PORT=$2; S0=$3; S1=$4; W=$5; PER=${6:-5} +RUN=${RUN:?set RUN= (results go to .sharpa_sim_eval//)} +OUT=.sharpa_sim_eval/$RUN/$TAG +LOG=.sharpa_sim_eval/logs/$RUN/$TAG; mkdir -p $OUT $LOG +until .venv_webxr/bin/python -c "import socket;socket.create_connection(('localhost',$PORT),2)" 2>/dev/null; do sleep 15; done +echo "[suite $TAG] server :$PORT up $(date -u +%FT%TZ)" +JOBS=$(.venv_webxr/bin/python - "$S0" "$S1" "$PER" <<'PY' +import json, sys +s0, s1, per = map(int, sys.argv[1:]) +t = json.load(open("benchmarks/sharpa_webxr/tasks.json")) +for sc in sorted(t, key=lambda k: -min(t[k]["max_steps"], 1200)): + for a in range(s0, s1 + 1, per): + print(f"{sc} {a}-{min(a + per - 1, s1)}") +PY +) +EXECUTE_STEPS=${EXECUTE_STEPS:-} # empty: the server's upstream-matched value +export OUT LOG PORT EXECUTE_STEPS MUJOCO_GL=osmesa LP_NUM_THREADS=4 +# Each worker exits 1 if its rollout or postprocess failed; xargs then exits nonzero (123) and so does this script. +echo "$JOBS" | xargs -P "$W" -L 1 bash -c ' + sc=$0; seeds=$1; tag=${sc:0:5}_${seeds}; rc=0 + .venv_webxr/bin/python -u benchmarks/sharpa_webxr/rollout.py --port $PORT --scene $sc --seeds $seeds --out $OUT ${EXECUTE_STEPS:+--execute-steps $EXECUTE_STEPS} >> $LOG/$tag.log 2>&1 \ + || { echo "[FAIL rollout] $sc $seeds" | tee -a $LOG/$tag.log; rc=1; } + a=${seeds%-*}; b=${seeds#*-}; dirs=$(for s in $(seq $a $b); do echo $OUT/$sc/seed_$s; done) + .venv_webxr/bin/python -u benchmarks/sharpa_webxr/postprocess.py $dirs --no-video --keyframes 16 >> $LOG/$tag.log 2>&1 \ + || { echo "[FAIL post] $sc $seeds" | tee -a $LOG/$tag.log; rc=1; } + echo "[job done] $sc $seeds rc=$rc $(date -u +%T)" + exit $rc +' +rc=$? +if [ $rc -ne 0 ]; then + echo "[suite $TAG] FAILED (xargs exit $rc): see [FAIL ...] lines above and $LOG/ $(date -u +%FT%TZ)" + exit 1 +fi +echo "[suite $TAG] ALL_DONE $(date -u +%FT%TZ)" diff --git a/benchmarks/sharpa_webxr/scenes.py b/benchmarks/sharpa_webxr/scenes.py new file mode 100644 index 00000000..33544f4e --- /dev/null +++ b/benchmarks/sharpa_webxr/scenes.py @@ -0,0 +1,93 @@ +"""Materialize published WebXR-Teleop scenes from the live GCS store into a local folder. + +Mirrors ``mujoco_webxr_teleop.cache.packages.materialize`` without the cache server/database: +``scenes//current.json`` -> revision manifest -> scene files + asset ZIPs (extracted under +``assets//``) + ``scene.xml.poses.json`` from the manifest layout. +""" +from __future__ import annotations + +import argparse +import hashlib +import io +import json +import subprocess +import zipfile +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +LIVE = "gs://foundational-research/webxr-teleop/live" +DEFAULT_ROOT = Path(__file__).resolve().parents[2] / "data" / "webxr_scenes" + + +def _cat(uri: str) -> bytes: + return subprocess.run(["gsutil", "-q", "cat", uri], check=True, capture_output=True).stdout + + +def list_scenes() -> list[str]: + out = subprocess.run(["gsutil", "ls", f"{LIVE}/scenes/"], check=True, capture_output=True, text=True).stdout + return sorted(line.rstrip("/").split("/")[-1] for line in out.split()) + + +def _within(base: Path, rel: str) -> Path: + """``base / rel`` resolved, or ValueError if it escapes ``base`` (absolute paths, ``..``, symlinks).""" + target = (base / rel).resolve() + if not target.is_relative_to(base.resolve()): + raise ValueError(f"unsafe path {rel!r}: resolves outside {base}") + return target + + +def _check_sha(data: bytes, want: str, what: str) -> None: + if hashlib.sha256(data).hexdigest() != want: + raise ValueError(f"{what}: sha256 mismatch") + + +def fetch_scene(scene_id: str, root: Path = DEFAULT_ROOT) -> Path: + """Download one scene (idempotent: a complete folder with a matching revision is reused).""" + folder = root / scene_id + current = json.loads(_cat(f"{LIVE}/scenes/{scene_id}/current.json")) + stamp = folder / ".revision.json" + if stamp.exists() and json.loads(stamp.read_text()).get("revision") == current["revision"]: + return folder + rev = f"{LIVE}/scenes/{scene_id}/revisions/{current['revision']}" + manifest_bytes = _cat(f"{rev}/manifest.json") + _check_sha(manifest_bytes, current["manifest_sha256"], f"{scene_id} manifest") + manifest = json.loads(manifest_bytes) + folder.mkdir(parents=True, exist_ok=True) + for name, ref in manifest["files"].items(): + target = _within(folder, name) # manifest names come from the store: never write outside the scene dir + data = _cat(f"{rev}/{name}") + _check_sha(data, ref["sha256"], f"{scene_id}/{name}") + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(data) + for ref in manifest.get("asset_packages", []): + blob = _cat(f"{LIVE}/{ref['key']}") + _check_sha(blob, ref["sha256"], ref["key"]) + dest = _within(folder / "assets", Path(ref["key"]).stem) + with zipfile.ZipFile(io.BytesIO(blob)) as z: + for member in z.infolist(): + if member.is_dir(): + continue + target = _within(dest, member.filename) # zip-slip: unconditional check (not an assert) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(z.read(member)) + if manifest.get("layout") is not None: + (folder / "scene.xml.poses.json").write_text(json.dumps(manifest["layout"])) + (folder / "scene_manifest.json").write_text(json.dumps(manifest, indent=1)) + stamp.write_text(json.dumps(current)) + return folder + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("scenes", nargs="*", help="scene ids (default: all live scenes)") + p.add_argument("--root", type=Path, default=DEFAULT_ROOT) + p.add_argument("--workers", type=int, default=8) + a = p.parse_args() + ids = a.scenes or list_scenes() + with ThreadPoolExecutor(a.workers) as ex: + for sid, path in zip(ids, ex.map(lambda s: fetch_scene(s, a.root), ids)): + print(sid, "->", path) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/sharpa_webxr/score.py b/benchmarks/sharpa_webxr/score.py new file mode 100644 index 00000000..b91308af --- /dev/null +++ b/benchmarks/sharpa_webxr/score.py @@ -0,0 +1,215 @@ +"""Score every rollout under a folder with a VLM judge and write per-task metadata into that folder. + +usage: score.py [--model gpt-6-luna] [--rubric r2] [--workers 8] [--limit N] [--force] + any directory containing rollout dirs (…//seed_k with rollout.json), e.g. + .sharpa_sim_eval/gwp05_full_v2/step005000_ema; searched recursively. + +Per rollout: +- r2 (default): the judge sees judge_frames/ (24 timesteps x 2 cameras at 480x640, written by + `postprocess.py --frames-only`) over the scored span (limits.limit: min(1.5 x p90, 20 s)) and grades + reach / grasp / move / complete / success with RUBRIC_R2 (grasp and move need visible lift or carry). +- r1: the 16 combined 320x240 keyframes and judge.py's rubric (first version, kept for comparison). +- The verdict is cached in /judge_[_r2].json; reruns skip it unless --force. + +Graded score follows GWP-0.5 report Sec. 4: 0.25 each for reach, grasp, move and complete (judge stages); success is +the judge's binary call. Sim signals (signals.json) are carried alongside, with judge/sim agreement per stage. + +Output, written into : +- scores_[_r2].json: overall and per-task (scene) SR, graded score, stage rates and per-seed rows +- scores_[_r2].tsv: one line per task + +Credentials: OPENAI_API_KEY, or the file ~/.config/openai/key (mode 600). +""" +from __future__ import annotations + +import argparse +import json +import math +import os +import sys +import time +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +import base64 + +import openai + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from judge import RUBRIC, SCHEMA, keyframes # noqa: E402 + +RUBRIC_R2 = """You grade robot manipulation rollouts from a physics simulator. A bimanual robot (two arms with +five-fingered dexterous hands) is given a natural-language task. You see frames sampled uniformly in time from one +rollout. At each time you get two images from fixed cameras: the HEAD camera (looking down at the table from above +the robot) and the CHEST camera (looking forward across the table). Captions give the time in seconds. + +Grade four stages, each true or false, from what the images show: +1. reach: a hand gets to the task object: touching it or within a few cm of it. +2. grasp: the hand holds the task object. True only if the object is visibly lifted off the table, or visibly + travels together with the hand between frames while the fingers are closed around it (for a lid, door, handle or + button: the hand is on it while it moves or is pressed). Fingers hovering over, touching, nudging or pushing the + object is NOT a grasp. +3. move: the task object is moved purposefully toward the task goal by the robot (carried, rotated, poured, opened, + closed), visible as a clear change of its position or state. Pushing it a little, knocking it over, or moving a + different object is NOT move. +4. complete: the task's goal state is visible: e.g. the object rests in/on the target, the lid is closed, the cap is + on. For tasks without a placement (shake, stir, toss, hit, grip and hold), the described action is clearly and + fully performed with the right object. + +success = the full task was accomplished at some time in the rollout. Be strict: when the images are ambiguous or +the object is too small to tell, answer false. A later stage normally requires the earlier ones; if not, say why in +the rationale. In the rationale, name the object you judged and what happened to it.""" + +STAGES = ("reach", "grasp", "move", "complete") +SIM_STAGES = ("reach", "grasp", "move") + + +def client() -> openai.OpenAI: + key = os.environ.get("OPENAI_API_KEY") + kf = Path.home() / ".config/openai/key" + if not key and kf.exists(): + key = kf.read_text().strip() + if not key: + sys.exit("no OpenAI key: set OPENAI_API_KEY or write it to ~/.config/openai/key (chmod 600)") + return openai.OpenAI(api_key=key, max_retries=6, timeout=300) + + +def frames_r2(d: Path) -> list[tuple[float, str, str]]: + """(t, head_b64, chest_b64) from judge_frames/t_.jpg.""" + fd = d / "judge_frames" + if not (fd / "COMPLETE").exists(): # written last by postprocess.judge_frames; partial frames would hide the end + raise FileNotFoundError(f"{fd}: incomplete judge frames; run postprocess.py --frames-only {d}") + ts = sorted({f.name.split("_")[0] for f in fd.glob("t*_ego_view.jpg")}) + if not ts: + raise FileNotFoundError(f"{fd}: no judge frames; run postprocess.py --frames-only {d}") + b64 = lambda f: base64.standard_b64encode(f.read_bytes()).decode() # noqa: E731 + return [(float(t[1:]), b64(fd / f"{t}_ego_view.jpg"), b64(fd / f"{t}_chest_view.jpg")) for t in ts] + + +def judge_one(c: openai.OpenAI, model: str, d: Path, force: bool, rubric: str = "r2") -> dict: + out = d / (f"judge_{model}.json" if rubric == "r1" else f"judge_{model}_{rubric}.json") + if out.exists() and not force: + return json.loads(out.read_text()) + meta = json.loads((d / "rollout.json").read_text()) + img = lambda b: {"type": "input_image", "image_url": f"data:image/jpeg;base64,{b}", "detail": "high"} # noqa: E731 + if rubric == "r1": + instructions = RUBRIC + content = [{"type": "input_text", "text": f"Task: {meta['task']}\nRollout length: {meta['steps'] / 20:.1f} s."}] + for t, b64 in keyframes(d): + content += [{"type": "input_text", "text": f"t = {t:.1f} s"}, + {"type": "input_image", "image_url": f"data:image/jpeg;base64,{b64}"}] + else: + instructions = RUBRIC_R2 + try: + fr = frames_r2(d) + except FileNotFoundError as e: + return {"error": str(e)} + content = [{"type": "input_text", "text": f"Task: {meta['task']}\nScored span: {fr[-1][0]:.1f} s, " + f"{len(fr)} times x 2 cameras."}] + for t, head, chest in fr: + content += [{"type": "input_text", "text": f"t = {t:.1f} s: head camera, then chest camera"}, img(head), img(chest)] + content.append({"type": "input_text", "text": "Grade this rollout."}) + t0 = time.time() + try: + resp = c.responses.create( + model=model, + instructions=instructions, + input=[{"role": "user", "content": content}], + text={"format": {"type": "json_schema", "name": "rollout_grade", "schema": SCHEMA, "strict": True}}, + ) + verdict = json.loads(resp.output_text) + verdict.update({"model": resp.model, "response_id": resp.id, + "usage": {"input": resp.usage.input_tokens, "output": resp.usage.output_tokens}}) + except (openai.APIError, json.JSONDecodeError) as e: # recorded, excluded from aggregates, retried on rerun + return {"error": f"{type(e).__name__}: {e}"[:500]} + verdict["latency_s"] = round(time.time() - t0, 2) + out.write_text(json.dumps(verdict, indent=1)) + return verdict + + +def wilson(k: int, n: int, z: float = 1.96): + if n == 0: + return None, None + p = k / n + den = 1 + z * z / n + c = (p + z * z / (2 * n)) / den + h = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / den + return round(c - h, 4), round(c + h, 4) + + +def summarize(rows: list[dict]) -> dict: + ok = [r for r in rows if "error" not in r["judge"]] + n = len(ok) + s = {"n": len(rows), "n_scored": n, "n_errors": len(rows) - n} + if not n: + return s + succ = sum(r["judge"]["success"] for r in ok) + s.update({"success_rate": round(succ / n, 4), "success_ci95": wilson(succ, n), + "graded": round(sum(r["graded"] for r in ok) / n, 4), + **{f"judge_{k}": round(sum(r["judge"][k] for r in ok) / n, 4) for k in STAGES}, + **{f"sim_{k}": round(sum(bool(r["sim"].get(k)) for r in ok) / n, 4) for k in SIM_STAGES}, + **{f"agree_{k}": round(sum(r["judge"][k] == bool(r["sim"].get(k)) for r in ok) / n, 4) for k in SIM_STAGES}}) + return s + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("folder", type=Path) + p.add_argument("--model", default="gpt-6-luna") + p.add_argument("--rubric", default="r2", choices=["r1", "r2"]) + p.add_argument("--workers", type=int, default=8) + p.add_argument("--limit", type=int, default=None, help="score only the first N rollouts (pilot / cost check)") + p.add_argument("--force", action="store_true", help="re-judge rollouts that already have a verdict") + a = p.parse_args() + + frames_dir = "judge_frames" if a.rubric == "r2" else "keyframes" + dirs = sorted(f.parent for f in a.folder.rglob("rollout.json") if (f.parent / frames_dir).is_dir()) + if a.limit: + dirs = dirs[: a.limit] + if not dirs: + sys.exit(f"no rollouts with {frames_dir}/ under {a.folder}" + (" (run postprocess.py --frames-only)" if a.rubric == "r2" else "")) + c = client() + print(f"judging {len(dirs)} rollouts under {a.folder} with {a.model}", flush=True) + + rows = [] + with ThreadPoolExecutor(a.workers) as ex: + for i, (d, v) in enumerate(zip(dirs, ex.map(lambda d: judge_one(c, a.model, d, a.force, a.rubric), dirs), strict=True)): + meta = json.loads((d / "rollout.json").read_text()) + sig = json.loads((d / "signals.json").read_text()) if (d / "signals.json").exists() else {} + row = {"scene_id": meta["scene_id"], "task": meta["task"], "seed": meta["seed"], "dir": str(d.relative_to(a.folder)), + "steps": meta["steps"], "judge": v, "sim": sig.get("stage", {}), "fallen": sig.get("fallen_any")} + if "error" not in v: + row["graded"] = 0.25 * sum(bool(v[k]) for k in STAGES) + rows.append(row) + tag = v.get("error") or f"success={v['success']} graded={row['graded']:.2f} ({v['confidence']})" + print(f"[{i + 1}/{len(dirs)}] {row['dir']}: {tag}", flush=True) + + tasks = {} + for r in rows: + tasks.setdefault(r["scene_id"], []).append(r) + per_task = {sc: {"task": rs[0]["task"], **summarize(rs), "seeds": sorted(rs, key=lambda r: r["seed"])} + for sc, rs in sorted(tasks.items())} + usage = [r["judge"].get("usage", {}) for r in rows if "error" not in r["judge"]] + out = {"folder": str(a.folder), "model": a.model, "rubric_version": a.rubric, "scored_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "rubric": "GWP-0.5 graded: 0.25 x (reach, grasp, move, complete) from the judge; success = judge binary", + "tokens": {"input": sum(u.get("input", 0) for u in usage), "output": sum(u.get("output", 0) for u in usage)}, + "overall": summarize(rows), + "task_mean": {"success_rate": round(sum(t.get("success_rate", 0) for t in per_task.values()) / len(per_task), 4), + "graded": round(sum(t.get("graded", 0) for t in per_task.values()) / len(per_task), 4)}, + "per_task": per_task} + suffix = a.model if a.rubric == "r1" else f"{a.model}_{a.rubric}" + jp = a.folder / f"scores_{suffix}.json" + jp.write_text(json.dumps(out, indent=1)) + cols = ["n", "n_scored", "success_rate", "graded", "judge_reach", "judge_grasp", "judge_move", "judge_complete", + "sim_reach", "sim_grasp", "sim_move"] + with open(a.folder / f"scores_{suffix}.tsv", "w") as f: + f.write("scene_id\ttask\t" + "\t".join(cols) + "\n") + for sc, t in per_task.items(): + f.write(f"{sc}\t{t['task']}\t" + "\t".join(str(t.get(k, "")) for k in cols) + "\n") + o = out["overall"] + print(f"\nwrote {jp}\noverall: n={o['n']} scored={o['n_scored']} errors={o['n_errors']} " + f"SR={o.get('success_rate')} CI={o.get('success_ci95')} graded={o.get('graded')} tokens={out['tokens']}") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/sharpa_webxr/task_breakdown.py b/benchmarks/sharpa_webxr/task_breakdown.py new file mode 100644 index 00000000..bcf5dc43 --- /dev/null +++ b/benchmarks/sharpa_webxr/task_breakdown.py @@ -0,0 +1,71 @@ +"""Per-task and per-category judge success (k/n) for Sharpa sim-eval score files, side by side across models. + + python benchmarks/sharpa_webxr/task_breakdown.py motus=path/scores_gpt-6-luna_r2.json xwam=... [--tasks] + +Each argument is