diff --git a/design/aaa.md b/design/accepted/aaa.md similarity index 100% rename from design/aaa.md rename to design/accepted/aaa.md diff --git a/design/boto-design.md b/design/accepted/boto-design.md similarity index 100% rename from design/boto-design.md rename to design/accepted/boto-design.md diff --git a/design/diagrams/boto-architecture.excalidraw b/design/accepted/diagrams/boto-architecture.excalidraw similarity index 100% rename from design/diagrams/boto-architecture.excalidraw rename to design/accepted/diagrams/boto-architecture.excalidraw diff --git a/design/diagrams/boto-architecture.png b/design/accepted/diagrams/boto-architecture.png similarity index 100% rename from design/diagrams/boto-architecture.png rename to design/accepted/diagrams/boto-architecture.png diff --git a/design/diagrams/botwo-design.excalidraw b/design/accepted/diagrams/botwo-design.excalidraw similarity index 100% rename from design/diagrams/botwo-design.excalidraw rename to design/accepted/diagrams/botwo-design.excalidraw diff --git a/design/diagrams/botwo-design.png b/design/accepted/diagrams/botwo-design.png similarity index 100% rename from design/diagrams/botwo-design.png rename to design/accepted/diagrams/botwo-design.png diff --git a/design/diagrams/hive_metastore_database_diagram.png b/design/accepted/diagrams/hive_metastore_database_diagram.png similarity index 100% rename from design/diagrams/hive_metastore_database_diagram.png rename to design/accepted/diagrams/hive_metastore_database_diagram.png diff --git a/design/diagrams/lakectl-testing-arch.excalidraw b/design/accepted/diagrams/lakectl-testing-arch.excalidraw similarity index 100% rename from design/diagrams/lakectl-testing-arch.excalidraw rename to design/accepted/diagrams/lakectl-testing-arch.excalidraw diff --git a/design/diagrams/lakectl-testing-arch.png b/design/accepted/diagrams/lakectl-testing-arch.png similarity index 100% rename from design/diagrams/lakectl-testing-arch.png rename to design/accepted/diagrams/lakectl-testing-arch.png diff --git a/design/diagrams/merge-refactor-design.excalidraw b/design/accepted/diagrams/merge-refactor-design.excalidraw similarity index 100% rename from design/diagrams/merge-refactor-design.excalidraw rename to design/accepted/diagrams/merge-refactor-design.excalidraw diff --git a/design/diagrams/merge-refactor-design.png b/design/accepted/diagrams/merge-refactor-design.png similarity index 100% rename from design/diagrams/merge-refactor-design.png rename to design/accepted/diagrams/merge-refactor-design.png diff --git a/design/diagrams/metastore-alt.excalidraw b/design/accepted/diagrams/metastore-alt.excalidraw similarity index 100% rename from design/diagrams/metastore-alt.excalidraw rename to design/accepted/diagrams/metastore-alt.excalidraw diff --git a/design/diagrams/metastore-alt.png b/design/accepted/diagrams/metastore-alt.png similarity index 100% rename from design/diagrams/metastore-alt.png rename to design/accepted/diagrams/metastore-alt.png diff --git a/design/diagrams/metastore.excalidraw b/design/accepted/diagrams/metastore.excalidraw similarity index 100% rename from design/diagrams/metastore.excalidraw rename to design/accepted/diagrams/metastore.excalidraw diff --git a/design/diagrams/metastore.png b/design/accepted/diagrams/metastore.png similarity index 100% rename from design/diagrams/metastore.png rename to design/accepted/diagrams/metastore.png diff --git a/design/diagrams/routerFS-by_lakefs.excalidraw b/design/accepted/diagrams/routerFS-by_lakefs.excalidraw similarity index 100% rename from design/diagrams/routerFS-by_lakefs.excalidraw rename to design/accepted/diagrams/routerFS-by_lakefs.excalidraw diff --git a/design/diagrams/routerFS-by_lakefs.png b/design/accepted/diagrams/routerFS-by_lakefs.png similarity index 100% rename from design/diagrams/routerFS-by_lakefs.png rename to design/accepted/diagrams/routerFS-by_lakefs.png diff --git a/design/diagrams/routerFs-s3.excalidraw b/design/accepted/diagrams/routerFs-s3.excalidraw similarity index 100% rename from design/diagrams/routerFs-s3.excalidraw rename to design/accepted/diagrams/routerFs-s3.excalidraw diff --git a/design/diagrams/routerFs_s3.png b/design/accepted/diagrams/routerFs_s3.png similarity index 100% rename from design/diagrams/routerFs_s3.png rename to design/accepted/diagrams/routerFs_s3.png diff --git a/design/diagrams/spark-with-interfacing-classloader.excalidraw b/design/accepted/diagrams/spark-with-interfacing-classloader.excalidraw similarity index 100% rename from design/diagrams/spark-with-interfacing-classloader.excalidraw rename to design/accepted/diagrams/spark-with-interfacing-classloader.excalidraw diff --git a/design/diagrams/spark-with-interfacing-classloader.png b/design/accepted/diagrams/spark-with-interfacing-classloader.png similarity index 100% rename from design/diagrams/spark-with-interfacing-classloader.png rename to design/accepted/diagrams/spark-with-interfacing-classloader.png diff --git a/design/double-rocksdbjni.md b/design/accepted/double-rocksdbjni.md similarity index 100% rename from design/double-rocksdbjni.md rename to design/accepted/double-rocksdbjni.md diff --git a/design/export-functionality.md b/design/accepted/export-functionality.md similarity index 100% rename from design/export-functionality.md rename to design/accepted/export-functionality.md diff --git a/design/lakectl-testing.md b/design/accepted/lakectl-testing.md similarity index 100% rename from design/lakectl-testing.md rename to design/accepted/lakectl-testing.md diff --git a/design/log-commits-per-path.md b/design/accepted/log-commits-per-path.md similarity index 100% rename from design/log-commits-per-path.md rename to design/accepted/log-commits-per-path.md diff --git a/design/merge-refactor.md b/design/accepted/merge-refactor.md similarity index 100% rename from design/merge-refactor.md rename to design/accepted/merge-refactor.md diff --git a/design/metastore-project.md b/design/accepted/metastore-project.md similarity index 100% rename from design/metastore-project.md rename to design/accepted/metastore-project.md diff --git a/design/metastore.md b/design/accepted/metastore.md similarity index 100% rename from design/metastore.md rename to design/accepted/metastore.md diff --git a/design/object-metadata-s3.md b/design/accepted/object-metadata-s3.md similarity index 100% rename from design/object-metadata-s3.md rename to design/accepted/object-metadata-s3.md diff --git a/design/playground.md b/design/accepted/playground.md similarity index 100% rename from design/playground.md rename to design/accepted/playground.md diff --git a/design/protected-branches.md b/design/accepted/protected-branches.md similarity index 100% rename from design/protected-branches.md rename to design/accepted/protected-branches.md diff --git a/design/spark-co-exist-with-object-storages.md b/design/accepted/spark-co-exist-with-object-storages.md similarity index 100% rename from design/spark-co-exist-with-object-storages.md rename to design/accepted/spark-co-exist-with-object-storages.md diff --git a/design/system_tests.md b/design/accepted/system_tests.md similarity index 100% rename from design/system_tests.md rename to design/accepted/system_tests.md diff --git a/design/task-management-api.md b/design/accepted/task-management-api.md similarity index 100% rename from design/task-management-api.md rename to design/accepted/task-management-api.md diff --git a/design/ui-testing.md b/design/accepted/ui-testing.md similarity index 100% rename from design/ui-testing.md rename to design/accepted/ui-testing.md diff --git a/design/open/metadata_kv/committer_flow.excalidraw b/design/open/metadata_kv/committer_flow.excalidraw new file mode 100644 index 00000000000..b01c5950927 --- /dev/null +++ b/design/open/metadata_kv/committer_flow.excalidraw @@ -0,0 +1,880 @@ +{ + "type": "excalidraw", + "version": 2, + "source": "https://excalidraw.com", + "elements": [ + { + "id": "r92Hd4FdSSrgZLdgTiY-G", + "type": "rectangle", + "x": 1197.7581515842014, + "y": 155.18120659722217, + "width": 279.7137451171875, + "height": 91.9537353515625, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "strokeSharpness": "sharp", + "seed": 1486763021, + "version": 163, + "versionNonce": 1914396867, + "isDeleted": false, + "boundElementIds": [ + "Vrydlj77tg4YvxQg6cnBp", + "Bi7g2mEqQx8N6Gkn2oCT_" + ] + }, + { + "id": "gQ8Dmx0P2X7KkxjLJO4L1", + "type": "text", + "x": 1259.613568250868, + "y": 189.6528795030382, + "width": 146, + "height": 29, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "strokeSharpness": "sharp", + "seed": 1486978083, + "version": 79, + "versionNonce": 560627139, + "isDeleted": false, + "boundElementIds": null, + "text": "User commits", + "fontSize": 22.84853515624999, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 20 + }, + { + "type": "rectangle", + "version": 181, + "versionNonce": 2093420589, + "isDeleted": false, + "id": "ck1GyPHSvH0Y4HDC3LuPs", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1190.329583062066, + "y": 305.5319892035591, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "width": 279.7137451171875, + "height": 91.9537353515625, + "seed": 1504352707, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElementIds": [] + }, + { + "type": "text", + "version": 457, + "versionNonce": 692861805, + "isDeleted": false, + "id": "kwHs1wDZq-2n4IUiVltU4", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1252.6131998697915, + "y": 315.22507126266896, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 163.71667480468741, + "height": 71.55828170271106, + "seed": 1597446125, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElementIds": [ + "Vrydlj77tg4YvxQg6cnBp", + "aUR15YY1zN-EA760rXKj5" + ], + "fontSize": 18.964057255030216, + "fontFamily": 1, + "text": "read from branch:\nstaging_token\n(amortized)", + "baseline": 64.55828170271106, + "textAlign": "center", + "verticalAlign": "top" + }, + { + "type": "rectangle", + "version": 252, + "versionNonce": 756426979, + "isDeleted": false, + "id": "3umuJsjwtbCAneU90kbKV", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1086.5425957573784, + "y": 454.408447265625, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "width": 508.8854302300347, + "height": 91.9537353515625, + "seed": 233672813, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElementIds": [ + "aUR15YY1zN-EA760rXKj5", + "XydvtKFFA35Dzr1p7muGR" + ] + }, + { + "type": "text", + "version": 699, + "versionNonce": 1906796067, + "isDeleted": false, + "id": "UAIQ7aWTYy_42iyOeOMtt", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1097.5842647359825, + "y": 479.9600020681162, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 487, + "height": 52, + "seed": 1036237027, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElementIds": [ + "XydvtKFFA35Dzr1p7muGR" + ], + "fontSize": 20.304426718592428, + "fontFamily": 1, + "text": "Push existing staging_token into sealed_tokens,\ncreate new staging_token (using SetIf)", + "baseline": 44, + "textAlign": "center", + "verticalAlign": "top" + }, + { + "type": "rectangle", + "version": 346, + "versionNonce": 1319632419, + "isDeleted": false, + "id": "ImUBvJb1gRD7lpeIt5nIo", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1178.195509168837, + "y": 590.8231133355035, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "width": 319.8420545789929, + "height": 89.97558593750003, + "seed": 1977134915, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElementIds": [ + "XydvtKFFA35Dzr1p7muGR" + ] + }, + { + "type": "text", + "version": 639, + "versionNonce": 126394627, + "isDeleted": false, + "id": "q-q8qKDlpO2oG_roWM3OD", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1195.0403713650173, + "y": 599.6402256137975, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 294, + "height": 75, + "seed": 1958681197, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElementIds": [ + "XydvtKFFA35Dzr1p7muGR", + "TTCeeTsYXDFW39GoniJ51", + "LUwFfR9QW8gKrwxgojZpt" + ], + "fontSize": 19.859373943421915, + "fontFamily": 1, + "text": "Iterate over \nall sealed_tokens,\napplying them to a new commit", + "baseline": 68, + "textAlign": "center", + "verticalAlign": "top" + }, + { + "id": "tGgFfaVfTCJ8yHvVJK3Cx", + "type": "diamond", + "x": 1196.984937879774, + "y": 899.6690741644965, + "width": 288.93109130859386, + "height": 132.9085693359375, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#fab005", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "strokeSharpness": "sharp", + "seed": 1538593219, + "version": 645, + "versionNonce": 1992081827, + "isDeleted": false, + "boundElementIds": [ + "unFlCnCc6DykYUzNMKf40", + "E3e__Vm6KN3Ebp2bPfBl1", + "GLXC-Gd9Fyz1tSOHpNgus", + "TTCeeTsYXDFW39GoniJ51", + "gueNUORdO2kN61G3QBRii" + ] + }, + { + "id": "DqTP_JKyQ0TnF9YzuqWiQ", + "type": "text", + "x": 1266.321838378906, + "y": 955.1143866644965, + "width": 157, + "height": 25, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "strokeSharpness": "sharp", + "seed": 744812931, + "version": 448, + "versionNonce": 1423259853, + "isDeleted": false, + "boundElementIds": null, + "text": "condition failed?", + "fontSize": 20, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 18 + }, + { + "id": "GLXC-Gd9Fyz1tSOHpNgus", + "type": "arrow", + "x": 1341.0969238281248, + "y": 1037.1080390082466, + "width": 0.8978271484375, + "height": 130.78668212890625, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "strokeSharpness": "round", + "seed": 1427615331, + "version": 1299, + "versionNonce": 399576205, + "isDeleted": false, + "boundElementIds": null, + "points": [ + [ + 0, + 0 + ], + [ + -0.8978271484375, + 130.78668212890625 + ] + ], + "lastCommittedPoint": null, + "startBinding": { + "elementId": "tGgFfaVfTCJ8yHvVJK3Cx", + "focus": -0.0009257391389537562, + "gap": 4.263574449134275 + }, + "endBinding": { + "elementId": "dirSv6aPsrM_9H1G6x_N7", + "focus": -0.014945824112813833, + "gap": 3.8770751953125 + }, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "DT2jHuG2SU4iYJlUSaF4P", + "type": "ellipse", + "x": 1739.0527479383675, + "y": 1138.6854248046873, + "width": 213.7161865234375, + "height": 112.7554931640625, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#82c91e", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "strokeSharpness": "sharp", + "seed": 950815939, + "version": 748, + "versionNonce": 1115995747, + "isDeleted": false, + "boundElementIds": [ + "E3e__Vm6KN3Ebp2bPfBl1", + "t8SLLsQVkrdGDb5HtdvQC" + ] + }, + { + "id": "8hpYuhjfBuZD3f-haiUmQ", + "type": "text", + "x": 1504.9363878038196, + "y": 931.9375406901041, + "width": 156, + "height": 25, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "strokeSharpness": "sharp", + "seed": 162885187, + "version": 670, + "versionNonce": 1384982925, + "isDeleted": false, + "boundElementIds": [], + "text": "No (happy path)", + "fontSize": 20, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "baseline": 18 + }, + { + "id": "Me0DrjhGndHI1fct_Ez2k", + "type": "text", + "x": 1768.316928439669, + "y": 1168.3676147460935, + "width": 151, + "height": 50, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "strokeSharpness": "sharp", + "seed": 115223789, + "version": 631, + "versionNonce": 1197326787, + "isDeleted": false, + "boundElementIds": null, + "text": "Done!\nReturn Success", + "fontSize": 20, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "baseline": 43 + }, + { + "id": "wjcYO6Vo6CHdjfAGZyvuo", + "type": "text", + "x": 1356.9238281249995, + "y": 1070.2941962348089, + "width": 331, + "height": 25, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "strokeSharpness": "sharp", + "seed": 1012166307, + "version": 465, + "versionNonce": 933916653, + "isDeleted": false, + "boundElementIds": null, + "text": "Yes (another commit taking place)", + "fontSize": 20, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "baseline": 18 + }, + { + "id": "dirSv6aPsrM_9H1G6x_N7", + "type": "rectangle", + "x": 1204.6545410156248, + "y": 1171.7717963324653, + "width": 274.47509765625, + "height": 98.050537109375, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "strokeSharpness": "sharp", + "seed": 121429357, + "version": 465, + "versionNonce": 1392348867, + "isDeleted": false, + "boundElementIds": [ + "GLXC-Gd9Fyz1tSOHpNgus", + "O7IDpnVj3UuUwpp1_QzAf", + "D-PuH98cVTaC2USXPhyog", + "4mfTmlgXzNXFcgZ-Rm-jJ" + ] + }, + { + "id": "Vrydlj77tg4YvxQg6cnBp", + "type": "arrow", + "x": 1342.8872654905483, + "y": 248.13494194878467, + "width": 0.7990782132148979, + "height": 54.01630316840283, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "strokeSharpness": "round", + "seed": 1949233997, + "version": 397, + "versionNonce": 570338989, + "isDeleted": false, + "boundElementIds": null, + "points": [ + [ + 0, + 0 + ], + [ + -0.7990782132148979, + 54.01630316840283 + ] + ], + "lastCommittedPoint": null, + "startBinding": { + "elementId": "r92Hd4FdSSrgZLdgTiY-G", + "focus": -0.04234047043510048, + "gap": 1 + }, + "endBinding": { + "elementId": "kwHs1wDZq-2n4IUiVltU4", + "focus": 0.08367705944043384, + "gap": 13.073826145481462 + }, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "aUR15YY1zN-EA760rXKj5", + "type": "arrow", + "x": 1338.1034414802666, + "y": 399.19140625, + "width": 0.8980845327691895, + "height": 54.06329345703125, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "strokeSharpness": "round", + "seed": 1312536035, + "version": 300, + "versionNonce": 516232323, + "isDeleted": false, + "boundElementIds": null, + "points": [ + [ + 0, + 0 + ], + [ + -0.8980845327691895, + 54.06329345703125 + ] + ], + "lastCommittedPoint": null, + "startBinding": { + "elementId": "kwHs1wDZq-2n4IUiVltU4", + "gap": 12.408053284619996, + "focus": -0.05815507573056247 + }, + "endBinding": { + "elementId": "3umuJsjwtbCAneU90kbKV", + "gap": 1.15374755859375, + "focus": -0.01787914982876982 + }, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "XydvtKFFA35Dzr1p7muGR", + "type": "arrow", + "x": 1342.916123116689, + "y": 547.094089846133, + "width": 0.5114080588298293, + "height": 45.13718494828788, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "strokeSharpness": "round", + "seed": 1931692397, + "version": 574, + "versionNonce": 1803796877, + "isDeleted": false, + "boundElementIds": null, + "points": [ + [ + 0, + 0 + ], + [ + -0.5114080588298293, + 45.13718494828788 + ] + ], + "lastCommittedPoint": null, + "startBinding": { + "elementId": "UAIQ7aWTYy_42iyOeOMtt", + "focus": -0.009425603290043612, + "gap": 15.134087778016806 + }, + "endBinding": { + "elementId": "q-q8qKDlpO2oG_roWM3OD", + "focus": -0.0009800140666402452, + "gap": 7.408950819376628 + }, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "Q54hi4frTY_6--oYxDpv_", + "type": "line", + "x": 1485.633273654514, + "y": 966.9921875, + "width": 342.38986545138914, + "height": 0, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#82c91e", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "strokeSharpness": "round", + "seed": 1963442947, + "version": 547, + "versionNonce": 1861512451, + "isDeleted": false, + "boundElementIds": null, + "points": [ + [ + 0, + 0 + ], + [ + 342.38986545138914, + 0 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": null + }, + { + "id": "t8SLLsQVkrdGDb5HtdvQC", + "type": "arrow", + "x": 1827.1583387586797, + "y": 967.979329427083, + "width": 2.0884070722765955, + "height": 161.23110120491765, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#82c91e", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "strokeSharpness": "round", + "seed": 1996353581, + "version": 1570, + "versionNonce": 276427501, + "isDeleted": false, + "boundElementIds": null, + "points": [ + [ + 0, + 0 + ], + [ + 2.0884070722765955, + 161.23110120491765 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": { + "elementId": "DT2jHuG2SU4iYJlUSaF4P", + "focus": -0.1479601877708326, + "gap": 10.131378952512797 + }, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "type": "rectangle", + "version": 441, + "versionNonce": 1887818915, + "isDeleted": false, + "id": "m3I-h0ozU7WdrOA4xxfaD", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1163.5724894205734, + "y": 746.6317409939238, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "width": 319.8420545789929, + "height": 89.97558593750003, + "seed": 1029072237, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElementIds": [ + "XydvtKFFA35Dzr1p7muGR", + "LUwFfR9QW8gKrwxgojZpt" + ] + }, + { + "type": "text", + "version": 924, + "versionNonce": 1349748899, + "isDeleted": false, + "id": "4tuo19q01V2pBHpf97x9q", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1182.4173516167536, + "y": 755.7315138624954, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 290, + "height": 75, + "seed": 901690339, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElementIds": [ + "XydvtKFFA35Dzr1p7muGR", + "TTCeeTsYXDFW39GoniJ51", + "gueNUORdO2kN61G3QBRii", + "LUwFfR9QW8gKrwxgojZpt" + ], + "fontSize": 19.859373943421915, + "fontFamily": 1, + "text": "Write Branch (SetIf):\nreplace commit ID\nremove applied sealed_tokens", + "baseline": 68, + "textAlign": "center", + "verticalAlign": "top" + }, + { + "type": "arrow", + "version": 714, + "versionNonce": 193995949, + "isDeleted": false, + "id": "LUwFfR9QW8gKrwxgojZpt", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1341.7826149162668, + "y": 683.6683866925226, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 0.10816167487132589, + "height": 61.02058793818367, + "seed": 1958463715, + "groupIds": [], + "strokeSharpness": "round", + "boundElementIds": [], + "startBinding": { + "elementId": "q-q8qKDlpO2oG_roWM3OD", + "focus": 0.002313441391067715, + "gap": 9.028161078725134 + }, + "endBinding": { + "elementId": "4tuo19q01V2pBHpf97x9q", + "focus": 0.10036412080143316, + "gap": 11.042539231789078 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 0.10816167487132589, + 61.02058793818367 + ] + ] + }, + { + "id": "h5blVu7SblDNdAsHCensf", + "type": "text", + "x": 1223.3920898437498, + "y": 1208.2970648871528, + "width": 237, + "height": 25, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#82c91e", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "strokeSharpness": "sharp", + "seed": 867032685, + "version": 168, + "versionNonce": 296082381, + "isDeleted": false, + "boundElementIds": null, + "text": "Retry or let caller know", + "fontSize": 20, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "baseline": 18 + }, + { + "id": "gueNUORdO2kN61G3QBRii", + "type": "arrow", + "x": 1339.5761578251397, + "y": 835.5658637152778, + "width": 1.267289028905907, + "height": 63.67372388513297, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#82c91e", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "strokeSharpness": "round", + "seed": 1975331907, + "version": 402, + "versionNonce": 1369562029, + "isDeleted": false, + "boundElementIds": null, + "points": [ + [ + 0, + 0 + ], + [ + -1.267289028905907, + 63.67372388513297 + ] + ], + "lastCommittedPoint": null, + "startBinding": { + "elementId": "4tuo19q01V2pBHpf97x9q", + "focus": -0.08920552787594868, + "gap": 4.834349852782452 + }, + "endBinding": { + "elementId": "tGgFfaVfTCJ8yHvVJK3Cx", + "focus": -0.030960966230291195, + "gap": 1.7030849033912077 + }, + "startArrowhead": null, + "endArrowhead": "arrow" + } + ], + "appState": { + "gridSize": null, + "viewBackgroundColor": "#ffffff" + } +} \ No newline at end of file diff --git a/design/open/metadata_kv/committer_flow.png b/design/open/metadata_kv/committer_flow.png new file mode 100644 index 00000000000..528aeaf3261 Binary files /dev/null and b/design/open/metadata_kv/committer_flow.png differ diff --git a/design/open/metadata_kv/index.md b/design/open/metadata_kv/index.md new file mode 100644 index 00000000000..b25423e7bec --- /dev/null +++ b/design/open/metadata_kv/index.md @@ -0,0 +1,247 @@ +# Design proposal - lakeFS on KV + +lakeFS stores 2 types of metadata: + +1. Immutable metadata, namely committed entries - stored as ranges and metaranges in the underlying object store) +2. Mutable metadata, namely IAM entities, branch pointers and uncommitted entries + +This proposal describes an alternative implementation for the **mutable** metadata. +Currently, this type of metadata is stored on (and relies on the guarantees provided by) a PostgreSQL database. + +In this proposal, we detail a relatively narrow database abstraction - one that could be satisfied by a wide variety of database systems. +We also detail how lakeFS operates on top of this abstraction, instead of relying on PostgreSQL specific behavior + +## Goals + +1. Make it easy to run lakeFS as a managed service in a multitenant environment +1. Allow running production lakeFS systems on top of a database that operations teams are capable of managing, as not all ops teams are versed in scaling PostgreSQL. +1. Increase users trust in lakeFS, in terms of positioning: PostgreSQL is apparently not the first DB that comes to mind in relation to scalability +1. Make lakeFS easier to experiment with: Allow running a local lakeFS server without any external dependencies (e.g. no more `docker-compose` in the quickstart) + +## Non-Goals + +1. Improve performance, latency or throughput of the system +1. Provide new features, capabilities or guarantees that weren't previously possible in the lakeFS API/UI/CLI + +## Design + +At the heart of the design, is a simple Key/Value interface, used for all mutable metadata management. + +### Semantics + +Order operations by a "happens before" operation: operation A happened before operation B if A +finished before B started. If operation A happened before B then we also say that B happened +after A. (As usual, these are not a total ordering!) + +#### Consistency guarantees + +* If a write succeeds, successive reads and listings will return the contents of either that + write or of some other write that did not happen before it. + +* A succesful read returns the contents of some write that did not happen after it. + +* A listing returns the contents of some writes that did not happen after it. + +* A successful commit holds the contents of some listing of the entire keyspace. + +* Mutating operations (commits and writes) may succeed or fail. When they fail, their contents + might still be visible. + +#### Consistency **non**-guarantees + +These guarantees do *not* give linearizability of any kind. In particular, these are some +possible behaviours. + +1. **Impossible ordering by application logic (the "1-3-2 problem"):** I write an application + that reads a number from a file, increments it, and writes back the same file (this + application performs an unsafe increment). I start with the file containing "1", and run the + application twice concurrently. An observer (some process that repeatedly reads the file) + may observe the value sequence "1", "3", "2". If the observer commits each version, it can + create a **history** of these values in this order. +2. **Non-monotonicity (the "B-A-N-A-N-A-N-A-... problem :banana:"):** A file has contents "B". + I start a continuous committer (some process that repeatedly commits). Now I run two + concurrent updates: one updates the file contents to "N", the other updates the file contents + to "A". Different orderings can cause histories that look like "B", "A", "N", "A", "N", "A", + "N", ... to any length. + +### Key/Value Store interface + +This is roughly the API: + +```go +type Store interface { + // Get returns a value for the given key, or ErrNotFound if key doesn't exist + Get(key []byte) (value []byte, err error) + // Scan returns an iterator that scans keys in byte order, starting at or after the `start` position + Scan(start []byte) (iter KeyValueIterator, err error) + // Set stores the given value, overwriting an existing value if one exists + Set(key, value []byte) error + // Delete will delete the key/value at key, if any + Delete(key []byte) error + // SetIf returns an ErrPredicateFailed error if the valuePredicate passed + // doesn't match the currently stored value. SetIf is a simple compare-and-swap operator: + // valuePredicate is either the existing value, or an opaque value representing it (hash, index, etc). + // this is intentianally simplistic: we can model a better abstraction on top, keeping this interface simple for implementors + SetIf(key, value, valuePredicate []byte) error +} +``` + +Note: This API is roughly the one needed and is subject to change/tweaking. +It is meant to illustrate the required capabilities in order to build a functioning lakeFS system on top. + +#### KV requirements + +- read-after-write consistency: a read that follows a successful write should return the written value or newer +- keys could be enumerated lexicographically, in ascending byte order +- supports a key-level conditional operation based on a current value - or essentially, allow modeling a CAS operator + +#### Databases that meet these requirements (Examples): + +- PostgreSQL +- MySQL +- Embedded Pebble/RocksDB (great option for a "quickstart" environment?) +- MongoDB +- AWS DynamoDB +- FoundationDB +- Azure Cosmos +- Azure Blob Store +- Google BigTable +- Google Spanner +- Google Cloud Storage +- HBase +- Cassandra (or compatible systems such as ScyllaDB) +- Raft (embedded, or server implementations such as Consul/ETCd) +- Persistent Redis (Redis Enterprise, AWS MemoryDB) +- Simple in-memory tree (for simplifying and speeding up tests?) + +### Data Modeling: IAM + +The API that exposes and manipulates IAM entities is already modeled as a lexicographically ordered key value store (pagination is based on sorted entity IDs, which are strings). + +Relationships are modeled as auxiliary keys. For example, modeling a user, a group and a membership would look something like this (pseudo code): + +```go +func WriteUser(kv Store, user User) error { + data := proto.MustMarshal(user) + kv.Set([]byte(fmt.Sprintf("iam/users/%s", user.ID)), data) +} + +func WriteGroup(kv Store, group Group) error { + data := proto.MustMarshal(group) + kv.Set([]byte(fmt.Sprintf("iam/groups/%s", group.ID)), data) +} + +func AddUserToGroup(kv Store, user User, group Group) error { + data := proto.MustMarshal(&Membership{GroupID: group.ID, UserID: user.ID}) + kv.Set([]byte(fmt.Sprintf("iam/user_groups/%s/%s", user.ID, group.ID)), data) +} + +func ListUserGroups(kv Store, userID string) []string { + groupIds := make([]string, 0) + prefix := []byte(fmt.Sprintf("iam/user_groups/%s/", user.ID)) + iter := kv.Scan(prefix) + for iter.Next() { + pair := iter.Pair() + if !bytes.HasPrefix(pair.Key(), prefix) { + break + } + membership := Membership{} + proto.MustUnmarshal(&membership) + groupIds = append(groupId, membership.GroupID) + } + ... + return groupIds +} + +``` + +It is possible to create a 2-way index for many-to-many relationships, but this is not generally required - it would be simpler to simply do a scan when needed and filter only the relevant values. +This is because the IAM keyspace is relatively very small - it would be surprising if the biggest lakeFS installation to ever exist would contain more than, say, 50k users. + +Some care does need to be applied when managing these secondary indices - for example, when deleting an entity, secondary indices need to be pruned first, to avoid inconsistencies. + +### Graveler Metadata - Commits, Tags, Repositories + +These are simpler entities - commits and tags are immutable and could potentially be stored on the object store and not in the key/value store (they are cheap either way, so it doesn't make much of a difference). +Repositories and tags, are also returned in lexicographical order, which map well to the suggested abstraction. Commits are usually returned using parent traversal, so no scanning takes place anyway. + +There are no special concurrency requirements for these entities, apart for last-write-wins which is already the case for all modern stores. + +### Graveler Metadata - Branches and Staged Writes + +This is where concurrency control gets interesting, and where lakeFS is expected to provide a **correct** system whose semantics are well understood (lakeFS currently [falls short](https://github.com/treeverse/lakeFS/issues/2405) in that regard). + +Concurrency is more of an issue here because of how a commit works: when a commit starts, it scans the currently staged changes, applies them to the current commit pointed to by the branch, updating the branch reference and removing the staged changes it applied. + +Getting this right means we have to take care of the following: + +1. Ensure all staged changes that finished successfully before the commit started are applied as part of the commit (causality) +1. Ensure acknowledged writes end up either in a resulting commit, or staged to be committed (no lost writes) + +To do this, we will employ 2 mechanisms: + +1. [Optimistic Concurrency Control](https://en.wikipedia.org/wiki/Optimistic_concurrency_control) on the branch pointer using `SetIf()` +1. Reliance on write [idempotency](https://en.wikipedia.org/wiki/Idempotence) provided by Graveler (i.e., Writing the same exact entry, with the same identity - will not appear as a change) + +This is what the proposed implementation will look like: + +#### Committer flow + +We add an additional field to each `Branch` object: In addition to the existing `staging_token`, we add an array of strings named `sealed_tokens`. + +1. get branch, find current `staging_token` +1. use `SetIf()` to update the branch (if not modified by another process): push existing `staging_token` into `sealed_tokens`, set new uuid as `staging_token`. The branch is assuming to be represented by a single key/value pair that contains the `staging_token`, `sealed_tokens` and `commit_id` fields. +1. take the list of sealed tokens, and using the [`CombinedIterator()`](https://github.com/treeverse/lakeFS/blob/master/pkg/graveler/combined_iterator.go#L11), turn them into a single iterator to be applied on top of the existing commit +1. Once the commit has been persisted (metaranges and ranges stored in object store, commit itself stored to KV using `Set()`), perform another `SetIf()` that updates the branch key/value pair again: replacing its commit ID with the new value, and clearing `sealed_tokens`, as these have materialized into the new commit. +1. If `SetIf()` fails, this means another commit is happening/has happened on the same branch. Can either retry or let the caller know. + +An important property here is delegating safety vs liveness to a single optimistic `SetIf()` operation: if a commit fails somewhere along the process, a subsequent commit would simply pick up where the failed one left off, +adding the current staging_token into the set of changes to apply. In an environment where there aren't many concurrent commits on the same branch, and that commits mostly succeed - the size of `sealed_tokens` would be relatively small. As an optimization, compaction strategies could be added to merge tokens together, but this *might not be necessary*, at least not for use cases we're familiar with. +This assumption must be tested - we should expose good metrics and log this information: how many `sealed_tokens` each branch holds. If we do see cases where this grows big, we might have to prioritize compaction. + +*Note: for branches that receive many frequent commits (i.e. streaming use cases) we can actually recognize contention: if we have a retry loop where we try to commit and fail because `SetIf()` returns an error, we can add some exponential backoff within the internal retry loop noted above. This could be helpful to help prevent "retry storms" where high contention results in even higher contention.* + +![Committer Flow](./committer_flow.png) + +#### Caching branch pointers and amortized reads + +In the current design, for each read/write operation we add a single amortized read of the branch record as well. +Let's define an "amortized read" as the act of batching requests for the same branch for a short duration, thus amortizing the DB lookup cost across those requests. + +For this design, we don't want to change this, at least for most requests: Add 1 additional wait time for a KV lookup that could be amortized across requests for the same branch. + +To do this, we introduce a small in-memory cache (can utilize the same caching mechanism that already exists for IAM). +Please note: this *does not violate consistency*, see the [Read flow](#reader-flow) and [Writer flow](writer-flow) below to understand how. + +#### Writer flow + +1. Read the branch's existing staging token: if branch exists in the cache, use it! Otherwise, do an amortized read (see [above](#caching-branch-pointers-and-amortized-reads)) and cache the result for a very short duration. +1. Write to the staging token received - this is another key/value record (e.g. `"graveler/staging/${repoId}/${stagingToken}/${path}"`) +1. Read the branch's existing staging token **again**. This is always an amortized read, not a cache read. If we get the same `staging_token` - great, no commit has *started while writing* the record, return success to the user. For a system with low contention between writes and commits, this will be the usual case. +1. If the `staging_token` *has* changed - **retry the operation**. If the previous write made it in time to be included in the commit, we'll end up writing a record with the same identity - an idempotent operation. + +![Writer Flow](./writer_flow.png) + +#### Reading/Listing flow + +1. Read the branch's existing staging token(s): if branch exists in the cache, use it! Otherwise, do an amortized read (see [above](#caching-branch-pointers-and-amortized-reads)) and cache the result for a very short duration. +1. The length of `sealed_list` will typically be *empty* or very small, see ((above)[#committer-flow]) +1. We now use the existing `CombinedIterator` to read through all staging tokens and underlying commit. +1. Read the branch's existing staging token(s) **again**. This is always an amortized read, not a cache read. If it hasn't changed - great, no commit has *started while reading* the record, return success to the user. For a system with low contention between writes and commits, this will be the usual case. +1. If it has changed, we're reading from a stale set of staging tokens. A committer might have already deleted records from it. Retry the process. + +![Reader Flow](./reader_flow.png) + +#### Important Note - exclusion duration + +It is important to understand that the current pessimistic approach locks the branch for the entire duration of the commit. +This takes time proportional to the amount of changes to be committted and is unbounded. All writes to a branch are blocked for that period of time. + +With the optimistic version, readers and writers end up retrying in during exactly 2 "constant time" operations during a commit: during the initial update with a new `staging_token`, and again when clearing `sealed_tokens`. +The duration of these operations is of course, not constant - it is (well, should be) very short, and not proportional to the size of the commit in any way. +### Open Questions + +1. Complexity cost - How complex is the system after implementing this? What would a real API look like? Serialization? +1. Performance - How does this affect critical path request latency? How does it affect overall system throughput? +1. Flexibility - Where could the narrow `Store` API become a constraint? What *won't* we be able to implement going forward due to lack of guarantees (e.g. no transactions)? +1. Alternatives - As this is also a solution to milestone #3, how does it fare against [known](https://github.com/treeverse/lakeFS/pull/1688) [proposals](https://github.com/treeverse/lakeFS/pull/1685)? diff --git a/design/open/metadata_kv/reader_flow.excalidraw b/design/open/metadata_kv/reader_flow.excalidraw new file mode 100644 index 00000000000..ffb55d28bc0 --- /dev/null +++ b/design/open/metadata_kv/reader_flow.excalidraw @@ -0,0 +1,896 @@ +{ + "type": "excalidraw", + "version": 2, + "source": "https://excalidraw.com", + "elements": [ + { + "type": "rectangle", + "version": 166, + "versionNonce": 441903479, + "isDeleted": false, + "id": "r92Hd4FdSSrgZLdgTiY-G", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1085.7991943359375, + "y": 164.7894287109375, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "width": 279.7137451171875, + "height": 91.9537353515625, + "seed": 1486763021, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElementIds": [ + "Vrydlj77tg4YvxQg6cnBp", + "Bi7g2mEqQx8N6Gkn2oCT_" + ] + }, + { + "type": "text", + "version": 100, + "versionNonce": 167759225, + "isDeleted": false, + "id": "gQ8Dmx0P2X7KkxjLJO4L1", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1131.3026416015625, + "y": 198.26629638671875, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 193, + "height": 29, + "seed": 1486978083, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElementIds": [], + "fontSize": 22.84853515624999, + "fontFamily": 1, + "text": "User reads entry", + "baseline": 20, + "textAlign": "center", + "verticalAlign": "top" + }, + { + "type": "rectangle", + "version": 184, + "versionNonce": 1892197015, + "isDeleted": false, + "id": "ck1GyPHSvH0Y4HDC3LuPs", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1083.5089721679688, + "y": 303.55377197265625, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "width": 279.7137451171875, + "height": 91.9537353515625, + "seed": 1504352707, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElementIds": [] + }, + { + "type": "text", + "version": 593, + "versionNonce": 299663769, + "isDeleted": false, + "id": "kwHs1wDZq-2n4IUiVltU4", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1085.8677062988281, + "y": 317.76840622360646, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 270, + "height": 69, + "seed": 1597446125, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElementIds": [ + "Vrydlj77tg4YvxQg6cnBp", + "aUR15YY1zN-EA760rXKj5", + "aS0wUY31aY5oTaqWJV91n" + ], + "fontSize": 17.959224967156477, + "fontFamily": 1, + "text": "read (cached) branch:\nstaging_token, sealed_tokens,\ncommit_id", + "baseline": 62, + "textAlign": "center", + "verticalAlign": "top" + }, + { + "type": "rectangle", + "version": 288, + "versionNonce": 1055274935, + "isDeleted": false, + "id": "3umuJsjwtbCAneU90kbKV", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1085.1296997070312, + "y": 451.58251953125, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "width": 279.7137451171875, + "height": 141.8032836914062, + "seed": 233672813, + "groupIds": [ + "Su0jlAcbdGSYjnRhdUEXT" + ], + "strokeSharpness": "sharp", + "boundElementIds": [ + "aUR15YY1zN-EA760rXKj5" + ] + }, + { + "type": "text", + "version": 775, + "versionNonce": 864256825, + "isDeleted": false, + "id": "UAIQ7aWTYy_42iyOeOMtt", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1126.85358914462, + "y": 468.8491557353252, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 192, + "height": 112, + "seed": 1036237027, + "groupIds": [ + "Su0jlAcbdGSYjnRhdUEXT" + ], + "strokeSharpness": "sharp", + "boundElementIds": [ + "XydvtKFFA35Dzr1p7muGR" + ], + "fontSize": 22.242914773615336, + "fontFamily": 1, + "text": "Read in order:\n1. staging_token\n2. sealed_tokens\n3. commit data", + "baseline": 104, + "textAlign": "left", + "verticalAlign": "top" + }, + { + "type": "rectangle", + "version": 485, + "versionNonce": 1523853527, + "isDeleted": false, + "id": "ImUBvJb1gRD7lpeIt5nIo", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1083.4676513671875, + "y": 642.6383666992188, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "width": 279.7137451171875, + "height": 91.9537353515625, + "seed": 1977134915, + "groupIds": [ + "09e1cNFNZuZGOq0wsQ8L8" + ], + "strokeSharpness": "sharp", + "boundElementIds": [ + "XydvtKFFA35Dzr1p7muGR" + ] + }, + { + "type": "text", + "version": 986, + "versionNonce": 176206873, + "isDeleted": false, + "id": "q-q8qKDlpO2oG_roWM3OD", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1098.13134765625, + "y": 655.8472636454815, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 250, + "height": 75, + "seed": 1958681197, + "groupIds": [ + "09e1cNFNZuZGOq0wsQ8L8" + ], + "strokeSharpness": "sharp", + "boundElementIds": [ + "TTCeeTsYXDFW39GoniJ51", + "SbK-WsUPORShgFP9bpAKx", + "XydvtKFFA35Dzr1p7muGR", + "wb4RcbAwD8ek8ey973np5" + ], + "fontSize": 19.859373943421915, + "fontFamily": 1, + "text": "read branch (amortized)\ncompare staging_token, \nsealed_tokens, commit_id", + "baseline": 68, + "textAlign": "center", + "verticalAlign": "top" + }, + { + "type": "ellipse", + "version": 1103, + "versionNonce": 1646720503, + "isDeleted": false, + "id": "DT2jHuG2SU4iYJlUSaF4P", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1146.5576036241318, + "y": 926.008080376519, + "strokeColor": "#000000", + "backgroundColor": "#82c91e", + "width": 149.62353515625, + "height": 83.761474609375, + "seed": 950815939, + "groupIds": [ + "VznIIABGIyosOyGvdhGPc" + ], + "strokeSharpness": "sharp", + "boundElementIds": [ + "E3e__Vm6KN3Ebp2bPfBl1", + "t8SLLsQVkrdGDb5HtdvQC", + "TvECZN296Kau4QTpDLLim" + ] + }, + { + "type": "text", + "version": 894, + "versionNonce": 1885653241, + "isDeleted": false, + "id": "Me0DrjhGndHI1fct_Ez2k", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1196.445075141059, + "y": 956.3991326226128, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 53, + "height": 25, + "seed": 115223789, + "groupIds": [ + "VznIIABGIyosOyGvdhGPc" + ], + "strokeSharpness": "sharp", + "boundElementIds": [], + "fontSize": 20, + "fontFamily": 1, + "text": "Done!", + "baseline": 18, + "textAlign": "center", + "verticalAlign": "middle" + }, + { + "type": "arrow", + "version": 666, + "versionNonce": 99280505, + "isDeleted": false, + "id": "Vrydlj77tg4YvxQg6cnBp", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1231.7818122662216, + "y": 257.7431640625, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 0.19587194209043446, + "height": 46.951416015625, + "seed": 1949233997, + "groupIds": [], + "strokeSharpness": "round", + "boundElementIds": [], + "startBinding": { + "elementId": "r92Hd4FdSSrgZLdgTiY-G", + "focus": -0.04234047043510048, + "gap": 1 + }, + "endBinding": { + "elementId": "kwHs1wDZq-2n4IUiVltU4", + "focus": 0.08367705944043384, + "gap": 13.073826145481462 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 0.19587194209043446, + 46.951416015625 + ] + ] + }, + { + "type": "arrow", + "version": 627, + "versionNonce": 949595993, + "isDeleted": false, + "id": "aUR15YY1zN-EA760rXKj5", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1226.7912574780719, + "y": 405.9030579400379, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 1.6390081350684795, + "height": 43.900249558834844, + "seed": 1312536035, + "groupIds": [], + "strokeSharpness": "round", + "boundElementIds": [], + "startBinding": { + "elementId": "kwHs1wDZq-2n4IUiVltU4", + "gap": 19.134651716431392, + "focus": -0.05815507573056247 + }, + "endBinding": { + "elementId": "3umuJsjwtbCAneU90kbKV", + "gap": 1.7792120323772889, + "focus": -0.01787914982876982 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -1.6390081350684795, + 43.900249558834844 + ] + ] + }, + { + "type": "arrow", + "version": 1051, + "versionNonce": 981279799, + "isDeleted": false, + "id": "XydvtKFFA35Dzr1p7muGR", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1221.4385035123948, + "y": 595.3309326171875, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 0, + "height": 45.5670510251075, + "seed": 1931692397, + "groupIds": [], + "strokeSharpness": "round", + "boundElementIds": [], + "startBinding": { + "elementId": "UAIQ7aWTYy_42iyOeOMtt", + "focus": 0.014740475335678125, + "gap": 14.481776881862288 + }, + "endBinding": { + "elementId": "q-q8qKDlpO2oG_roWM3OD", + "focus": -0.013542753150841235, + "gap": 14.949280003186459 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 0, + 45.5670510251075 + ] + ] + }, + { + "type": "arrow", + "version": 1234, + "versionNonce": 310353593, + "isDeleted": false, + "id": "TvECZN296Kau4QTpDLLim", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1223.7429266779527, + "y": 879.6757935113792, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 0.16161054355461602, + "height": 43.42787499250426, + "seed": 1151154989, + "groupIds": [], + "strokeSharpness": "round", + "boundElementIds": [], + "startBinding": { + "elementId": "_EcOpSsCUGvCkWS5XT6di", + "focus": 2.5528856610936197, + "gap": 15.599473766925712 + }, + "endBinding": { + "elementId": "DT2jHuG2SU4iYJlUSaF4P", + "focus": 0.027339001835420918, + "gap": 2.922329810052169 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -0.16161054355461602, + 43.42787499250426 + ] + ] + }, + { + "id": "YN32VG26ebuWjmzfE0CLe", + "type": "diamond", + "x": 1109.7294243706597, + "y": 787.04099867079, + "width": 230.75177001953125, + "height": 91.508544921875, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#fab005", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "strokeSharpness": "sharp", + "seed": 509168951, + "version": 133, + "versionNonce": 1274051991, + "isDeleted": false, + "boundElementIds": [ + "TvECZN296Kau4QTpDLLim", + "9AyENa_Tninv0Dh6lBr1u", + "15g-Q2qQJZihOfO6FnJGx" + ] + }, + { + "id": "kxSu-QubjhOFUZa3Q3vga", + "type": "text", + "x": 1147.1053093804253, + "y": 820.2952711317275, + "width": 156, + "height": 25, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#82c91e", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "strokeSharpness": "sharp", + "seed": 1794932535, + "version": 111, + "versionNonce": 727269655, + "isDeleted": false, + "boundElementIds": null, + "text": "branch changed?", + "fontSize": 20, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "baseline": 18 + }, + { + "id": "_EcOpSsCUGvCkWS5XT6di", + "type": "text", + "x": 1239.3424004448784, + "y": 880.1106703016494, + "width": 20, + "height": 25, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "strokeSharpness": "sharp", + "seed": 1373626873, + "version": 57, + "versionNonce": 894222967, + "isDeleted": false, + "boundElementIds": [ + "TvECZN296Kau4QTpDLLim" + ], + "text": "no", + "fontSize": 20, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "baseline": 18 + }, + { + "id": "SbK-WsUPORShgFP9bpAKx", + "type": "arrow", + "x": 1226.587395562066, + "y": 736.0090620252822, + "width": 0, + "height": 51.74406433105469, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "strokeSharpness": "round", + "seed": 4055065, + "version": 50, + "versionNonce": 615515257, + "isDeleted": false, + "boundElementIds": null, + "points": [ + [ + 0, + 0 + ], + [ + 0, + 51.74406433105469 + ] + ], + "lastCommittedPoint": null, + "startBinding": { + "elementId": "q-q8qKDlpO2oG_roWM3OD", + "focus": -0.027648383246527377, + "gap": 5.161798379800757 + }, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "uQ9-_tiWBWcQzTCWWpPkK", + "type": "line", + "x": 1110.8633355034722, + "y": 834.631773206923, + "width": 134.79325441904348, + "height": 0, + "angle": 0, + "strokeColor": "#343a40", + "backgroundColor": "#ced4da", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "strokeSharpness": "round", + "seed": 95661015, + "version": 45, + "versionNonce": 1024134489, + "isDeleted": false, + "boundElementIds": null, + "points": [ + [ + 0, + 0 + ], + [ + -134.79325441904348, + 0 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": null + }, + { + "id": "eIDMuMuYmz7C6gh65z9AP", + "type": "line", + "x": 981.6175774468317, + "y": 833.6015760633683, + "width": 0, + "height": 490.9214782714844, + "angle": 0, + "strokeColor": "#343a40", + "backgroundColor": "#ced4da", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "strokeSharpness": "round", + "seed": 230745527, + "version": 160, + "versionNonce": 922341559, + "isDeleted": false, + "boundElementIds": null, + "points": [ + [ + 0, + 0 + ], + [ + 0, + -490.9214782714844 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": null + }, + { + "id": "aS0wUY31aY5oTaqWJV91n", + "type": "arrow", + "x": 982.8823784722223, + "y": 344.3671858045792, + "width": 101.98532782660584, + "height": 0, + "angle": 0, + "strokeColor": "#343a40", + "backgroundColor": "#ced4da", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "strokeSharpness": "round", + "seed": 1879302071, + "version": 107, + "versionNonce": 2057195191, + "isDeleted": false, + "boundElementIds": null, + "points": [ + [ + 0, + 0 + ], + [ + 101.98532782660584, + 0 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": { + "elementId": "kwHs1wDZq-2n4IUiVltU4", + "focus": 0.229020881710935, + "gap": 1 + }, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "wb4RcbAwD8ek8ey973np5", + "type": "arrow", + "x": 1366.4005974663628, + "y": 697.1320020887589, + "width": 95.81207275390625, + "height": 0, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "strokeSharpness": "round", + "seed": 1819214679, + "version": 23, + "versionNonce": 520921559, + "isDeleted": false, + "boundElementIds": null, + "points": [ + [ + 0, + 0 + ], + [ + 95.81207275390625, + 0 + ] + ], + "lastCommittedPoint": null, + "startBinding": { + "elementId": "q-q8qKDlpO2oG_roWM3OD", + "focus": 0.10092635848739821, + "gap": 18.269249810112797 + }, + "endBinding": { + "elementId": "T5Pp4j68FZIuXv4FWp9qV", + "focus": 0.0723680280371971, + "gap": 2.59307861328125 + }, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "type": "rectangle", + "version": 219, + "versionNonce": 518540057, + "isDeleted": false, + "id": "T5Pp4j68FZIuXv4FWp9qV", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1464.8057488335503, + "y": 654.4823896620011, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "width": 279.7137451171875, + "height": 91.9537353515625, + "seed": 746887383, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElementIds": [ + "Vrydlj77tg4YvxQg6cnBp", + "Bi7g2mEqQx8N6Gkn2oCT_", + "wb4RcbAwD8ek8ey973np5" + ] + }, + { + "id": "0cwHpaM18RRKSSBlZSYd7", + "type": "text", + "x": 1502.162621392144, + "y": 675.7771742078995, + "width": 205, + "height": 50, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "strokeSharpness": "sharp", + "seed": 2049570455, + "version": 60, + "versionNonce": 856394617, + "isDeleted": false, + "boundElementIds": null, + "text": "Cache branch\nfor a short duration", + "fontSize": 20, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "baseline": 43 + }, + { + "id": "D0C0q42n96j8uZQfJ7Vwq", + "type": "text", + "x": 1026.7428521050347, + "y": 797.2410413953995, + "width": 32, + "height": 25, + "angle": 0, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "strokeSharpness": "sharp", + "seed": 1679166937, + "version": 45, + "versionNonce": 971913721, + "isDeleted": false, + "boundElementIds": null, + "text": "Yes", + "fontSize": 20, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "baseline": 18 + }, + { + "id": "K3U96WIrfJ9EuxM3MusoR", + "type": "text", + "x": 643.3885125054254, + "y": 631.821958753798, + "width": 311, + "height": 125, + "angle": 0, + "strokeColor": "#495057", + "backgroundColor": "#ced4da", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "strokeSharpness": "sharp", + "seed": 623853943, + "version": 351, + "versionNonce": 1945049337, + "isDeleted": false, + "boundElementIds": null, + "text": "if staging token(s) changes\nwhile reading, some records\nmay have already been deleted.\n\nSo we try again", + "fontSize": 20, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "baseline": 118 + } + ], + "appState": { + "gridSize": null, + "viewBackgroundColor": "#ffffff" + } +} \ No newline at end of file diff --git a/design/open/metadata_kv/reader_flow.png b/design/open/metadata_kv/reader_flow.png new file mode 100644 index 00000000000..db0d9aa68de Binary files /dev/null and b/design/open/metadata_kv/reader_flow.png differ diff --git a/design/open/metadata_kv/writer_flow.excalidraw b/design/open/metadata_kv/writer_flow.excalidraw new file mode 100644 index 00000000000..72422726367 --- /dev/null +++ b/design/open/metadata_kv/writer_flow.excalidraw @@ -0,0 +1,933 @@ +{ + "type": "excalidraw", + "version": 2, + "source": "https://excalidraw.com", + "elements": [ + { + "type": "rectangle", + "version": 167, + "versionNonce": 222392680, + "isDeleted": false, + "id": "r92Hd4FdSSrgZLdgTiY-G", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1085.7991943359375, + "y": 164.7894287109375, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "width": 279.7137451171875, + "height": 91.9537353515625, + "seed": 1486763021, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElementIds": [ + "Vrydlj77tg4YvxQg6cnBp", + "Bi7g2mEqQx8N6Gkn2oCT_" + ] + }, + { + "type": "text", + "version": 101, + "versionNonce": 1993783576, + "isDeleted": false, + "id": "gQ8Dmx0P2X7KkxjLJO4L1", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1128.9827270507812, + "y": 198.26629638671875, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 197.6398291015625, + "height": 28.560668945312504, + "seed": 1486978083, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElementIds": [], + "fontSize": 22.84853515624999, + "fontFamily": 1, + "text": "User writes entry", + "baseline": 19.560668945312504, + "textAlign": "center", + "verticalAlign": "middle" + }, + { + "type": "rectangle", + "version": 185, + "versionNonce": 299828328, + "isDeleted": false, + "id": "ck1GyPHSvH0Y4HDC3LuPs", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1083.5089721679688, + "y": 303.55377197265625, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "width": 279.7137451171875, + "height": 91.9537353515625, + "seed": 1504352707, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElementIds": [] + }, + { + "type": "text", + "version": 693, + "versionNonce": 1749156968, + "isDeleted": false, + "id": "kwHs1wDZq-2n4IUiVltU4", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1090.3081665039062, + "y": 320.9544863531295, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 258.751220703125, + "height": 65.36872944078948, + "seed": 1597446125, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElementIds": [ + "Vrydlj77tg4YvxQg6cnBp", + "aUR15YY1zN-EA760rXKj5" + ], + "fontSize": 17.217448997273767, + "fontFamily": 1, + "text": "read (cached) branch:\nstaging_token, sealed_tokens,\ncommit_id", + "baseline": 59.36872944078948, + "textAlign": "center", + "verticalAlign": "top" + }, + { + "type": "rectangle", + "version": 289, + "versionNonce": 1934767976, + "isDeleted": false, + "id": "3umuJsjwtbCAneU90kbKV", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1085.1296997070312, + "y": 451.58251953125, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "width": 279.7137451171875, + "height": 91.9537353515625, + "seed": 233672813, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElementIds": [ + "aUR15YY1zN-EA760rXKj5", + "XydvtKFFA35Dzr1p7muGR" + ] + }, + { + "type": "text", + "version": 776, + "versionNonce": 1268601624, + "isDeleted": false, + "id": "UAIQ7aWTYy_42iyOeOMtt", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1095.0501708984375, + "y": 468.8491557353252, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 255.60683649236503, + "height": 55.95373535156249, + "seed": 1036237027, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElementIds": [], + "fontSize": 22.242914773615336, + "fontFamily": 1, + "text": "write entry\nas staging_token+path", + "baseline": 47.95373535156249, + "textAlign": "center", + "verticalAlign": "top" + }, + { + "type": "rectangle", + "version": 486, + "versionNonce": 1172769384, + "isDeleted": false, + "id": "ImUBvJb1gRD7lpeIt5nIo", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1083.4334106445312, + "y": 579.1428833007812, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "width": 279.7137451171875, + "height": 91.9537353515625, + "seed": 1977134915, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElementIds": [] + }, + { + "type": "text", + "version": 987, + "versionNonce": 934490136, + "isDeleted": false, + "id": "q-q8qKDlpO2oG_roWM3OD", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1135.5330200195312, + "y": 591.0685160868877, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 174.88427734375003, + "height": 75.39920939371255, + "seed": 1958681197, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElementIds": [ + "XydvtKFFA35Dzr1p7muGR", + "TTCeeTsYXDFW39GoniJ51" + ], + "fontSize": 19.859373943421915, + "fontFamily": 1, + "text": "read from branch:\nstaging_token\n(again. amortized)", + "baseline": 68.39920939371255, + "textAlign": "center", + "verticalAlign": "top" + }, + { + "type": "diamond", + "version": 238, + "versionNonce": 1956822925, + "isDeleted": false, + "id": "tGgFfaVfTCJ8yHvVJK3Cx", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1075.5394897460938, + "y": 701.6465454101562, + "strokeColor": "#000000", + "backgroundColor": "#fab005", + "width": 288.93109130859386, + "height": 132.9085693359375, + "seed": 1538593219, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElementIds": [ + "unFlCnCc6DykYUzNMKf40", + "E3e__Vm6KN3Ebp2bPfBl1", + "GLXC-Gd9Fyz1tSOHpNgus" + ] + }, + { + "type": "text", + "version": 30, + "versionNonce": 1724191971, + "isDeleted": false, + "id": "DqTP_JKyQ0TnF9YzuqWiQ", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1119.8113403320312, + "y": 757.0918579101562, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 206, + "height": 25, + "seed": 744812931, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElementIds": [], + "fontSize": 20, + "fontFamily": 1, + "text": "Got different token?", + "baseline": 18, + "textAlign": "center", + "verticalAlign": "middle" + }, + { + "type": "arrow", + "version": 93, + "versionNonce": 1686302509, + "isDeleted": false, + "id": "GLXC-Gd9Fyz1tSOHpNgus", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1219.08642578125, + "y": 839.0855102539062, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 0.8978271484375, + "height": 130.78668212890625, + "seed": 1427615331, + "groupIds": [], + "strokeSharpness": "round", + "boundElementIds": [], + "startBinding": { + "elementId": "tGgFfaVfTCJ8yHvVJK3Cx", + "focus": 0.0029855734901953657, + "gap": 4.499712368808829 + }, + "endBinding": { + "elementId": "dirSv6aPsrM_9H1G6x_N7", + "focus": -0.014945824112813833, + "gap": 3.8770751953125 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -0.8978271484375, + 130.78668212890625 + ] + ] + }, + { + "type": "ellipse", + "version": 1104, + "versionNonce": 879328616, + "isDeleted": false, + "id": "DT2jHuG2SU4iYJlUSaF4P", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1426.3644883897568, + "y": 939.5325249565972, + "strokeColor": "#000000", + "backgroundColor": "#82c91e", + "width": 213.7161865234375, + "height": 112.7554931640625, + "seed": 950815939, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElementIds": [ + "E3e__Vm6KN3Ebp2bPfBl1", + "t8SLLsQVkrdGDb5HtdvQC" + ] + }, + { + "type": "text", + "version": 267, + "versionNonce": 254999085, + "isDeleted": false, + "id": "8hpYuhjfBuZD3f-haiUmQ", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1382.9258897569446, + "y": 733.9150119357639, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 156, + "height": 25, + "seed": 162885187, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElementIds": [ + "t8SLLsQVkrdGDb5HtdvQC" + ], + "fontSize": 20, + "fontFamily": 1, + "text": "No (happy path)", + "baseline": 18, + "textAlign": "left", + "verticalAlign": "top" + }, + { + "type": "text", + "version": 895, + "versionNonce": 1057320216, + "isDeleted": false, + "id": "Me0DrjhGndHI1fct_Ez2k", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1504.628668891059, + "y": 981.7147148980034, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 53, + "height": 25, + "seed": 115223789, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElementIds": [], + "fontSize": 20, + "fontFamily": 1, + "text": "Done!", + "baseline": 18, + "textAlign": "center", + "verticalAlign": "middle" + }, + { + "type": "text", + "version": 32, + "versionNonce": 1933526797, + "isDeleted": false, + "id": "wjcYO6Vo6CHdjfAGZyvuo", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1234.913330078125, + "y": 872.2716674804688, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 207, + "height": 25, + "seed": 1012166307, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElementIds": [], + "fontSize": 20, + "fontFamily": 1, + "text": "Yes (commit started)", + "baseline": 18, + "textAlign": "left", + "verticalAlign": "top" + }, + { + "type": "rectangle", + "version": 60, + "versionNonce": 1984942189, + "isDeleted": false, + "id": "dirSv6aPsrM_9H1G6x_N7", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1082.64404296875, + "y": 973.749267578125, + "strokeColor": "#000000", + "backgroundColor": "#ced4da", + "width": 274.47509765625, + "height": 98.050537109375, + "seed": 121429357, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElementIds": [ + "GLXC-Gd9Fyz1tSOHpNgus", + "O7IDpnVj3UuUwpp1_QzAf", + "D-PuH98cVTaC2USXPhyog", + "4mfTmlgXzNXFcgZ-Rm-jJ" + ] + }, + { + "type": "text", + "version": 6, + "versionNonce": 1501524141, + "isDeleted": false, + "id": "B7CDzRaY8-1F_VrzlS9C8", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1192.881591796875, + "y": 1010.2745361328125, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 54, + "height": 25, + "seed": 901114307, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElementIds": [], + "fontSize": 20, + "fontFamily": 1, + "text": "Retry", + "baseline": 18, + "textAlign": "center", + "verticalAlign": "middle" + }, + { + "type": "arrow", + "version": 867, + "versionNonce": 89470824, + "isDeleted": false, + "id": "Vrydlj77tg4YvxQg6cnBp", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1231.346804366117, + "y": 257.7431640625, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 0.42560782715986534, + "height": 50.13749614514802, + "seed": 1949233997, + "groupIds": [], + "strokeSharpness": "round", + "boundElementIds": [], + "startBinding": { + "elementId": "r92Hd4FdSSrgZLdgTiY-G", + "focus": -0.04234047043510048, + "gap": 1 + }, + "endBinding": { + "elementId": "kwHs1wDZq-2n4IUiVltU4", + "focus": 0.08367705944043384, + "gap": 13.073826145481462 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -0.42560782715986534, + 50.13749614514802 + ] + ] + }, + { + "type": "arrow", + "version": 828, + "versionNonce": 529140328, + "isDeleted": false, + "id": "aUR15YY1zN-EA760rXKj5", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1225.811699401232, + "y": 398.73126907853896, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 1.7542558149914385, + "height": 51.69750289411735, + "seed": 1312536035, + "groupIds": [], + "strokeSharpness": "round", + "boundElementIds": [], + "startBinding": { + "elementId": "kwHs1wDZq-2n4IUiVltU4", + "focus": -0.05815507573056247, + "gap": 12.408053284619996 + }, + "endBinding": { + "elementId": "3umuJsjwtbCAneU90kbKV", + "focus": -0.01787914982876982, + "gap": 1.15374755859375 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -1.7542558149914385, + 51.69750289411735 + ] + ] + }, + { + "type": "arrow", + "version": 1052, + "versionNonce": 552026984, + "isDeleted": false, + "id": "XydvtKFFA35Dzr1p7muGR", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1224.9820556640625, + "y": 547.7094116210938, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 3.5606689453125, + "height": 28.48553466796875, + "seed": 1931692397, + "groupIds": [], + "strokeSharpness": "round", + "boundElementIds": [], + "startBinding": { + "elementId": "3umuJsjwtbCAneU90kbKV", + "focus": -0.04302215983449187, + "gap": 4.17315673828125 + }, + "endBinding": { + "elementId": "q-q8qKDlpO2oG_roWM3OD", + "focus": -0.08817118363864009, + "gap": 14.873569797825269 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -3.5606689453125, + 28.48553466796875 + ] + ] + }, + { + "type": "arrow", + "version": 41, + "versionNonce": 1694526861, + "isDeleted": false, + "id": "TTCeeTsYXDFW39GoniJ51", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1220.5697021484375, + "y": 673.2590942382812, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 0.5086669921875, + "height": 30.77435302734375, + "seed": 1711642125, + "groupIds": [], + "strokeSharpness": "round", + "boundElementIds": [], + "startBinding": { + "elementId": "q-q8qKDlpO2oG_roWM3OD", + "focus": 0.03566496122726051, + "gap": 6.791368757680914 + }, + "endBinding": null, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 0.5086669921875, + 30.77435302734375 + ] + ] + }, + { + "type": "line", + "version": 21, + "versionNonce": 1436459277, + "isDeleted": false, + "id": "C-T6106kYaOy6uYFSueAv", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 0, + "opacity": 100, + "angle": 0, + "x": 1075.4878743489585, + "y": 1030.6480577256946, + "strokeColor": "#000000", + "backgroundColor": "#82c91e", + "width": 71.83085123697924, + "height": 0, + "seed": 144527011, + "groupIds": [ + "NBereUWIXtuasW6ju29pf" + ], + "strokeSharpness": "round", + "boundElementIds": [], + "startBinding": null, + "endBinding": null, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": null, + "points": [ + [ + 0, + 0 + ], + [ + -71.83085123697924, + 0 + ] + ] + }, + { + "type": "line", + "version": 60, + "versionNonce": 1486944109, + "isDeleted": false, + "id": "cIh2HZGsDk7yMrpsVy1be", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 0, + "opacity": 100, + "angle": 0, + "x": 1001.5443929036459, + "y": 1028.9751519097222, + "strokeColor": "#000000", + "backgroundColor": "#82c91e", + "width": 0, + "height": 914.4569905598958, + "seed": 1373733443, + "groupIds": [ + "NBereUWIXtuasW6ju29pf" + ], + "strokeSharpness": "round", + "boundElementIds": [], + "startBinding": null, + "endBinding": null, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": null, + "points": [ + [ + 0, + 0 + ], + [ + 0, + -914.4569905598958 + ] + ] + }, + { + "type": "line", + "version": 53, + "versionNonce": 1237942733, + "isDeleted": false, + "id": "KxM_a9EdHo-gj_HgZ5aFY", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 0, + "opacity": 100, + "angle": 0, + "x": 1002.8462727864584, + "y": 115.58614095052087, + "strokeColor": "#000000", + "backgroundColor": "#82c91e", + "width": 223.18983289930554, + "height": 0, + "seed": 933118435, + "groupIds": [ + "NBereUWIXtuasW6ju29pf" + ], + "strokeSharpness": "round", + "boundElementIds": [], + "startBinding": null, + "endBinding": null, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": null, + "points": [ + [ + 0, + 0 + ], + [ + 223.18983289930554, + 0 + ] + ] + }, + { + "type": "arrow", + "version": 28, + "versionNonce": 1848754221, + "isDeleted": false, + "id": "Bi7g2mEqQx8N6Gkn2oCT_", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 0, + "opacity": 100, + "angle": 0, + "x": 1226.9269476996528, + "y": 114.36964246961804, + "strokeColor": "#000000", + "backgroundColor": "#82c91e", + "width": 0, + "height": 49.4538709852431, + "seed": 853675395, + "groupIds": [ + "NBereUWIXtuasW6ju29pf" + ], + "strokeSharpness": "round", + "boundElementIds": [], + "startBinding": null, + "endBinding": { + "elementId": "r92Hd4FdSSrgZLdgTiY-G", + "focus": 0.009087010040133254, + "gap": 1 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 0, + 49.4538709852431 + ] + ] + }, + { + "type": "line", + "version": 92, + "versionNonce": 1177170883, + "isDeleted": false, + "id": "Q54hi4frTY_6--oYxDpv_", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "x": 1363.622775607639, + "y": 768.9696587456598, + "strokeColor": "#000000", + "backgroundColor": "#82c91e", + "width": 151.36583116319457, + "height": 0, + "seed": 1963442947, + "groupIds": [], + "strokeSharpness": "round", + "boundElementIds": [], + "startBinding": null, + "endBinding": null, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": null, + "points": [ + [ + 0, + 0 + ], + [ + 151.36583116319457, + 0 + ] + ] + }, + { + "type": "arrow", + "version": 213, + "versionNonce": 1738421123, + "isDeleted": false, + "id": "t8SLLsQVkrdGDb5HtdvQC", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "x": 1514.4700792100696, + "y": 768.826429578993, + "strokeColor": "#000000", + "backgroundColor": "#82c91e", + "width": 2.0884070722765955, + "height": 161.23110120491765, + "seed": 1996353581, + "groupIds": [], + "strokeSharpness": "round", + "boundElementIds": [], + "startBinding": { + "elementId": "8hpYuhjfBuZD3f-haiUmQ", + "focus": -0.6813279815569624, + "gap": 9.911417643229129 + }, + "endBinding": { + "elementId": "DT2jHuG2SU4iYJlUSaF4P", + "focus": -0.14796018777082917, + "gap": 10.153136274070533 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 2.0884070722765955, + 161.23110120491765 + ] + ] + } + ], + "appState": { + "gridSize": null, + "viewBackgroundColor": "#ffffff" + } +} \ No newline at end of file diff --git a/design/open/metadata_kv/writer_flow.png b/design/open/metadata_kv/writer_flow.png new file mode 100644 index 00000000000..575f8ad6461 Binary files /dev/null and b/design/open/metadata_kv/writer_flow.png differ