overhaul site

Author: b5

config.toml

@@ -14,36 +14,34 @@ highlight_code = true
 
 [extra]
 # Put all your custom variables here
-
 [[extra.docs_sidebar]]
-section = "iroh"
+section = "Iroh"
 links = [
- { title = "Overview", path = "/docs" },
- { title = "Install", path = "/docs/install" },
-#  { title = "Quickstart", path = "/docs/quickstart" },
- { title = "Services", path = "/docs/services" },
-#  { title = "Metrics", path = "/docs/metrics" },
- { title = "Concepts", path = "/docs/concepts" },
- { title = "Iroh & Kubo (go-IPFS)", path = "/docs/iroh-and-kubo" },
- { title = "Use Cases", path = "/docs/use-cases" },
+  { title = "Overview", path = "/docs/"},
+  { title = "Install", path = "/docs/install" },
+  { title = "CLI", path = "/docs/cli" },
 ]
 
 [[extra.docs_sidebar]]
-section = "platforms"
+section = "Beetle"
 links = [
-  { title = "Overview", path = "/docs/platforms" },
-  { title = "cloud", path = "/docs/platforms/cloud" },
-  { title = "mobile", path = "/docs/platforms/mobile" },
+  { title = "Overview", path = "/docs/beetle" },
+  { title = "Install", path = "/docs/beetle/install" },
+  { title = "Services", path = "/docs/beetle/services" },
+  { title = "CLI", path = "/docs/beetle/cli" },
+  { title = "Data Locations", path = "/docs/beetle/data-locations" },
+  { title = "Beetle & Kubo (go-IPFS)", path = "/docs/beetle/beetle-and-kubo" },
 ]
 
-[[extra.docs_sidebar]]
-section = "reference"
+[[extra.design_docs_sidebar]]
+section = "Iroh Design Documents"
 links = [
-  { title = "Data Locations", path = "/docs/data-locations" },
-  { title = "CLI", path = "/docs/cli" },
-  # { title = "configuration", path = "/docs/config" },
-  # { title = "config: store", path = "/docs/config/store" },
-  # { title = "config: p2p", path = "/docs/config/p2p" },
-  # { title = "config: gateway", path = "/docs/config/gateway" },
-  # { title = "config: cli", path = "/docs/config/cli" },
+  { title = "0. Introduction", path = "/design" },
+  { title = "1. Iroh", path = "/design/iroh" },
+  { title = "2. Content Addressing", path = "/design/content-addressing" },
+  { title = "3. DSHT", path = "/design/dsht" },
+  { title = "4. Data Transfer", path = "/design/data-transfer" },
+  # { title = "concepts", path = "/design/concepts" },
+  # { title = "FAQ", path = "/design/faq" },
+  # { title = "For IPFS Veterans", path = "/design/for-ipfs-veterans" },
 ]

content/design/_index.md

@@ -0,0 +1,35 @@
++++
+title = "Introduction"
+description = ""
+template="design/page.html"
+[extra]
+section="0."
++++
+
+# Design Documents
+
+These _design documents_ describe at a high level how we are building Iroh. Think of them as a constantly evolving whitepaper: Design documents should help you reason about how iroh might work one day.
+
+**These design documents are a work in progress. They change often, integrating feedback & research as we go.** Each document ends with a changelog. A fine-grained history is available [on github](https://github.com/n0-computer/iroh.compute). Our goal with design documents is to present a single source of truth for our research. We gather feedback from numerous places ([github discussions](https://github.com/n0-computer/iroh/discussions), [twitter](https://twitter.com/n0computer), [meetings](https://www.youtube.com/watch?v=XMLiq9d50Fs), [chats](https://discord.com/invite/ipfs), etc.) and integreate them here.
+
+## These are not specs
+
+We're reserving the word _specs_ for finished, detailed descriptions of how Iroh actually works. You should be able to write an interoperable implementation of Iroh from specs. Iroh is currently pre-1.0 sofware, and has no formalized specs. 
+
+## These are not docs
+
+Our [docs](/docs) describe released versions of Iroh. If you want an accurate description of actual working software, head there.
+
+
+## Discussion
+
+The best way to influence these documents is to share your views on the Iroh [github discussion](https://github.com/n0-computer/iroh) forum. Numerous conversations are already in progress there, please search before posting a new topic.
+
+## Evaluation
+
+The only way to know if this design works is to build it, test it, put it into production and measure the results. We work hard to take a measurements-focused approach to designing distributed systems. **Our [perf suite](https://perf.iroh.computer) is the primary set of figures we use to evalaute designs.** The results we gather from continual experimentation directly impacts these design documents.
+
+
+<a class="next-page-button" href="/design/iroh">
+Next: 1. Iroh
+</a>

content/design/concepts.md

@@ -0,0 +1,19 @@
++++
+title = "concepts"
+description = "common data types & concepts used throughout iroh"
+template="design/page.html"
+[extra]
+section="design"
++++
+
+# Concepts
+IPFS intrduces a number of new terms & techniques. We outline some of the more prominent ones here for reference.
+
+## Content IDentifier (CID)
+Content IDentifiers look like this:
+
+```
+bafybeihjgu5w6wbbxqevdgccj5xm453dbzpkwmkyoepvs3vh6wft4uvf2q
+```
+
+A CID contains the _hash_ of it's contents, which is: the result of running the content through a hash function. The CID for different things will always be a different set of characters. Once content is in IPFS, we refer to it by the CID. A great tool for learning about CIDs is the [CID inspector](https://cid.ipfs.tech). Iroh only produces CIDs that use the BLAKE3 hashing function.

content/design/content-addressing/_index.md

@@ -0,0 +1,42 @@
++++
+title = "Content Addressing"
+description = ""
+template="design/page.html"
+[extra]
+section="2."
++++
+
+## Content Addressed Blobs
+
+Iroh works with _blobs_ of opaque data, which are often the bytes of a file. Blobs can be of arbitrary size ranging from 0-u64MAX bytes.
+
+All blobs within Iroh are referred to by the BLAKE3 [3] hash of contents. BLAKE3 is a _tree hashing_ algorithm, that splits its input into uniform chunks and arranges them as the leaves of a binary tree, producing intermediate _chunk hashes_ that accumulate up to a _root hash._ Iroh uses the 32 byte root hash (or just “hash”) as an immutable blob identifier. 
+
+<img class="figure" src="/design/content-addressing/fig_1_blob.svg" />
+
+We leverage the tree hash structure of BLAKE3 as a basis for incremental verification when data is sent over the network, as described by Section 6.4 “Verified Streaming” in [3] and implemented in [4]. Iroh caches all chunk hashes as external metadata, leaving the unaltered input blob as the canonical source of bytes. Verified streaming also facilitates _range requests:_ fetching a verifiable contiguous subsequence of a blob by streaming only the portions of the BLAKE3 binary tree required to verify the designated subsequence.
+
+Chunk hashes are distinct from root hashes, and only used during data transfer. The chunk size of BLAKE3 is a tunable constant that defaults to 1KiB, which results in a 6% overhead on on the size of the input blob. Increasing the chunk size reduces overhead, at the cost of requiring more data transfer before an incremental verification checkpoint is reached. The chunk size constant can be modified & recalculated *without affecting the root hash.* This opens the door to experiment with different chunk size constants, even at runtime. We intend to investigate chunk size optimization in future work.
+
+Root hashes are expressed as a Content identifier (CID) as defined in [5], making Iroh an *IPFS system* capable of interoperating with other systems that use CIDs. In contrast to other IPFS systems, only root hashes are valid content identifiers, which enforces a strict 1-1 relationship between a Content Identifier and blob. This 1-1 relationship brings Iroh into alignment with common whole-file checksum systems. A naive implementation of Iroh can skip verified streaming entirely and use the the CID as a whole-file checksum.
+
+## Collections
+
+A *Collection* is an ordered set of named *links* to blobs. Collection sizes can range from 0-billions of links. Collections are the only means of relating blobs within iroh, and form the basis of synchronization & querying. Collections are true sets, and cannot be nested to form graphs.
+
+A link is a triple of `name, size, CID`. *Name* is an opaque, arbitrary sequence of bytes that labels the link, typically a UTF-8 string. Names must be unique across the collection, and order the set. *size* is the length of the blob in bytes, and *CID* is the content identifier for the linked blob.
+
+Collections have an optional _header_ section that stores the number of items in the collection and  offsets to items within the list, forming a skip list index.
+
+<img class="figure" src="/design/content-addressing/fig_2_collection.svg" />
+
+Collections are content addressed in the exact same manner as blobs, but use differing CID multicodec identifier to distinguish them from opaque blobs. Like blobs, collections can be seeked into using byte offsets.
+
+## Collection Querying
+
+Queries can be performed to animate collection sets into graph structures. A typical example is constructing directories using slash separators combined with a prefix query. We have yet to begin researching collection querying and will have more to report in the future.
+
+
+<a class="next-page-button" href="/design/dsht">
+Next: 3. Distributed Sloppy Hash Table
+</a>