Add comprehensive installation and setup documentation

- Add GETTING_STARTED.md with quick start guide and development modes
- Add INSTALL.sh automated installation script
- Add INSTALLATION_CHECKLIST.md, INSTALLATION_SUCCESS.md, and INSTALLATION_SUMMARY.md
- Add QUICK_REFERENCE.md for common commands
- Add SETUP_GUIDE.md with detailed setup instructions
- Update README.md with improved project overview
- Add did-wallet app dependencies and node_modules
This commit is contained in:
Dorian
2026-01-27 17:18:21 +00:00
parent a81f655133
commit 0d073fa89e
22658 changed files with 4494151 additions and 6 deletions
+4
View File
@@ -0,0 +1,4 @@
This project is dual licensed under MIT and Apache-2.0.
MIT: https://www.opensource.org/licenses/mit
Apache-2.0: https://www.apache.org/licenses/license-2.0
+266
View File
@@ -0,0 +1,266 @@
# ipfs-unixfs-exporter <!-- omit in toc -->
[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech)
[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech)
[![codecov](https://img.shields.io/codecov/c/github/ipfs/js-ipfs-unixfs.svg?style=flat-square)](https://codecov.io/gh/ipfs/js-ipfs-unixfs)
[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/js-ipfs-unixfs/js-test-and-release.yml?branch=master\&style=flat-square)](https://github.com/ipfs/js-ipfs-unixfs/actions/workflows/js-test-and-release.yml?query=branch%3Amaster)
> JavaScript implementation of the UnixFs exporter used by IPFS
## Table of contents <!-- omit in toc -->
- [Install](#install)
- [Browser `<script>` tag](#browser-script-tag)
- [Example](#example)
- [API](#api)
- [`exporter(cid, blockstore, options)`](#exportercid-blockstore-options)
- [UnixFSEntry](#unixfsentry)
- [Raw entries](#raw-entries)
- [CBOR entries](#cbor-entries)
- [`entry.content({ offset, length })`](#entrycontent-offset-length-)
- [`walkPath(cid, blockstore)`](#walkpathcid-blockstore)
- [`recursive(cid, blockstore)`](#recursivecid-blockstore)
- [API Docs](#api-docs)
- [License](#license)
- [Contribute](#contribute)
## Install
```console
$ npm i ipfs-unixfs-exporter
```
### Browser `<script>` tag
Loading this module through a script tag will make it's exports available as `IpfsUnixfsExporter` in the global namespace.
```html
<script src="https://unpkg.com/ipfs-unixfs-exporter/dist/index.min.js"></script>
```
## Example
```js
// import a file and export it again
import { importer } from 'ipfs-unixfs-importer'
import { exporter } from 'ipfs-unixfs-exporter'
import { MemoryBlockstore } from 'blockstore-core/memory'
// Should contain the blocks we are trying to export
const blockstore = new MemoryBlockstore()
const files = []
for await (const file of importer([{
path: '/foo/bar.txt',
content: new Uint8Array([0, 1, 2, 3])
}], blockstore)) {
files.push(file)
}
console.info(files[0].cid) // Qmbaz
const entry = await exporter(files[0].cid, blockstore)
console.info(entry.cid) // Qmqux
console.info(entry.path) // Qmbaz/foo/bar.txt
console.info(entry.name) // bar.txt
console.info(entry.unixfs.fileSize()) // 4
// stream content from unixfs node
const size = entry.unixfs.fileSize()
const bytes = new Uint8Array(size)
let offset = 0
for await (const buf of entry.content()) {
bytes.set(buf, offset)
offset += chunk.length
}
console.info(bytes) // 0, 1, 2, 3
```
## API
```js
import { exporter } from 'ipfs-unixfs-exporter'
```
### `exporter(cid, blockstore, options)`
Uses the given [blockstore][] instance to fetch an IPFS node by it's CID.
Returns a Promise which resolves to a `UnixFSEntry`.
`options` is an optional object argument that might include the following keys:
- `signal` ([AbortSignal](https://developer.mozilla.org/en-US/docs/Web/API/AbortSignal)): Used to cancel any network requests that are initiated as a result of this export
### UnixFSEntry
```javascript
{
type: 'file' // or 'directory'
name: 'foo.txt',
path: 'Qmbar/foo.txt',
cid: CID, // see https://github.com/multiformats/js-cid
content: function, // returns an async iterator
unixfs: UnixFS // see https://github.com/ipfs/js-ipfs-unixfs
}
```
If the entry is a file, `entry.content()` returns an async iterator that yields one or more Uint8Arrays containing the file content:
```javascript
if (entry.type === 'file') {
for await (const chunk of entry.content()) {
// chunk is a Buffer
}
}
```
If the entry is a directory, `entry.content()` returns further `entry` objects:
```javascript
if (entry.type === 'directory') {
for await (const entry of dir.content()) {
console.info(entry.name)
}
}
```
### Raw entries
Entries with a `raw` codec `CID` return raw entries:
```javascript
{
name: 'foo.txt',
path: 'Qmbar/foo.txt',
cid: CID, // see https://github.com/multiformats/js-cid
node: Buffer, // see https://nodejs.org/api/buffer.html
content: function, // returns an async iterator
}
```
`entry.content()` returns an async iterator that yields a buffer containing the node content:
```javascript
for await (const chunk of entry.content()) {
// chunk is a Buffer
}
```
Unless you an options object containing `offset` and `length` keys as an argument to `entry.content()`, `chunk` will be equal to `entry.node`.
### CBOR entries
Entries with a `dag-cbor` codec `CID` return JavaScript object entries:
```javascript
{
name: 'foo.txt',
path: 'Qmbar/foo.txt',
cid: CID, // see https://github.com/multiformats/js-cid
node: Uint8Array,
content: function // returns an async iterator that yields a single object - see https://github.com/ipld/js-ipld-dag-cbor
}
```
There is no `content` function for a `CBOR` node.
### `entry.content({ offset, length })`
When `entry` is a file or a `raw` node, `offset` and/or `length` arguments can be passed to `entry.content()` to return slices of data:
```javascript
const length = 5
const data = new Uint8Array(length)
let offset = 0
for await (const chunk of entry.content({
offset: 0,
length
})) {
data.set(chunk, offset)
offset += chunk.length
}
// `data` contains the first 5 bytes of the file
return data
```
If `entry` is a directory, passing `offset` and/or `length` to `entry.content()` will limit the number of files returned from the directory.
```javascript
const entries = []
for await (const entry of dir.content({
offset: 0,
length: 5
})) {
entries.push(entry)
}
// `entries` contains the first 5 files/directories in the directory
```
### `walkPath(cid, blockstore)`
`walkPath` will return an async iterator that yields entries for all segments in a path:
```javascript
import { walkPath } from 'ipfs-unixfs-exporter'
const entries = []
for await (const entry of walkPath('Qmfoo/foo/bar/baz.txt', blockstore)) {
entries.push(entry)
}
// entries contains 4x `entry` objects
```
### `recursive(cid, blockstore)`
`recursive` will return an async iterator that yields all entries beneath a given CID or IPFS path, as well as the containing directory.
```javascript
import { recursive } from 'ipfs-unixfs-exporter'
const entries = []
for await (const child of recursive('Qmfoo/foo/bar', blockstore)) {
entries.push(entry)
}
// entries contains all children of the `Qmfoo/foo/bar` directory and it's children
```
## API Docs
- <https://ipfs.github.io/js-ipfs-unixfs/modules/ipfs_unixfs_exporter.html>
## License
Licensed under either of
- Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / <http://www.apache.org/licenses/LICENSE-2.0>)
- MIT ([LICENSE-MIT](LICENSE-MIT) / <http://opensource.org/licenses/MIT>)
## Contribute
Contributions welcome! Please check out [the issues](https://github.com/ipfs/js-ipfs-unixfs/issues).
Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general.
Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md).
Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions.
[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md)
[dag API]: https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/DAG.md
[blockstore]: https://github.com/ipfs/js-ipfs-interfaces/tree/master/packages/interface-blockstore#readme
[UnixFS]: https://github.com/ipfs/specs/tree/master/unixfs
File diff suppressed because one or more lines are too long
+100
View File
@@ -0,0 +1,100 @@
import { CID } from 'multiformats/cid';
import type { PBNode } from '@ipld/dag-pb';
import type { Bucket } from 'hamt-sharding';
import type { Blockstore } from 'interface-blockstore';
import type { UnixFS } from 'ipfs-unixfs';
import type { ProgressOptions, ProgressEvent } from 'progress-events';
export interface ExportProgress {
/**
* How many bytes of the file have been read
*/
bytesRead: bigint;
/**
* How many bytes of the file will be read - n.b. this may be
* smaller than `fileSize` if `offset`/`length` have been
* specified
*/
totalBytes: bigint;
/**
* The size of the file being read - n.b. this may be
* larger than `total` if `offset`/`length` has been
* specified
*/
fileSize: bigint;
}
export interface ExportWalk {
cid: CID;
}
/**
* Progress events emitted by the exporter
*/
export type ExporterProgressEvents = ProgressEvent<'unixfs:exporter:progress:unixfs:file', ExportProgress> | ProgressEvent<'unixfs:exporter:progress:unixfs:raw', ExportProgress> | ProgressEvent<'unixfs:exporter:progress:raw', ExportProgress> | ProgressEvent<'unixfs:exporter:progress:identity', ExportProgress> | ProgressEvent<'unixfs:exporter:walk:file', ExportWalk> | ProgressEvent<'unixfs:exporter:walk:directory', ExportWalk> | ProgressEvent<'unixfs:exporter:walk:hamt-sharded-directory', ExportWalk> | ProgressEvent<'unixfs:exporter:walk:raw', ExportWalk>;
export interface ExporterOptions extends ProgressOptions<ExporterProgressEvents> {
offset?: number;
length?: number;
signal?: AbortSignal;
}
export interface Exportable<T> {
type: 'file' | 'directory' | 'object' | 'raw' | 'identity';
name: string;
path: string;
cid: CID;
depth: number;
size: bigint;
content: (options?: ExporterOptions) => AsyncGenerator<T, void, unknown>;
}
export interface UnixFSFile extends Exportable<Uint8Array> {
type: 'file';
unixfs: UnixFS;
node: PBNode;
}
export interface UnixFSDirectory extends Exportable<UnixFSEntry> {
type: 'directory';
unixfs: UnixFS;
node: PBNode;
}
export interface ObjectNode extends Exportable<any> {
type: 'object';
node: Uint8Array;
}
export interface RawNode extends Exportable<Uint8Array> {
type: 'raw';
node: Uint8Array;
}
export interface IdentityNode extends Exportable<Uint8Array> {
type: 'identity';
node: Uint8Array;
}
export type UnixFSEntry = UnixFSFile | UnixFSDirectory | ObjectNode | RawNode | IdentityNode;
export interface NextResult {
cid: CID;
name: string;
path: string;
toResolve: string[];
}
export interface ResolveResult {
entry: UnixFSEntry;
next?: NextResult;
}
export interface Resolve {
(cid: CID, name: string, path: string, toResolve: string[], depth: number, blockstore: ReadableStorage, options: ExporterOptions): Promise<ResolveResult>;
}
export interface Resolver {
(cid: CID, name: string, path: string, toResolve: string[], resolve: Resolve, depth: number, blockstore: ReadableStorage, options: ExporterOptions): Promise<ResolveResult>;
}
export type UnixfsV1FileContent = AsyncIterable<Uint8Array> | Iterable<Uint8Array>;
export type UnixfsV1DirectoryContent = AsyncIterable<UnixFSEntry> | Iterable<UnixFSEntry>;
export type UnixfsV1Content = UnixfsV1FileContent | UnixfsV1DirectoryContent;
export interface UnixfsV1Resolver {
(cid: CID, node: PBNode, unixfs: UnixFS, path: string, resolve: Resolve, depth: number, blockstore: ReadableStorage): (options: ExporterOptions) => UnixfsV1Content;
}
export interface ShardTraversalContext {
hamtDepth: number;
rootBucket: Bucket<boolean>;
lastBucket: Bucket<boolean>;
}
export type ReadableStorage = Pick<Blockstore, 'get'>;
export declare function walkPath(path: string | CID, blockstore: ReadableStorage, options?: ExporterOptions): AsyncGenerator<UnixFSEntry, void, any>;
export declare function exporter(path: string | CID, blockstore: ReadableStorage, options?: ExporterOptions): Promise<UnixFSEntry>;
export declare function recursive(path: string | CID, blockstore: ReadableStorage, options?: ExporterOptions): AsyncGenerator<UnixFSEntry, void, any>;
//# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,GAAG,EAAE,MAAM,kBAAkB,CAAA;AAEtC,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,cAAc,CAAA;AAC1C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,eAAe,CAAA;AAC3C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACtD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAA;AACzC,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAA;AAErE,MAAM,WAAW,cAAc;IAC7B;;OAEG;IACH,SAAS,EAAE,MAAM,CAAA;IAEjB;;;;OAIG;IACH,UAAU,EAAE,MAAM,CAAA;IAElB;;;;OAIG;IACH,QAAQ,EAAE,MAAM,CAAA;CACjB;AAED,MAAM,WAAW,UAAU;IACzB,GAAG,EAAE,GAAG,CAAA;CACT;AAED;;GAEG;AACH,MAAM,MAAM,sBAAsB,GAChC,aAAa,CAAC,sCAAsC,EAAE,cAAc,CAAC,GACrE,aAAa,CAAC,qCAAqC,EAAE,cAAc,CAAC,GACpE,aAAa,CAAC,8BAA8B,EAAE,cAAc,CAAC,GAC7D,aAAa,CAAC,mCAAmC,EAAE,cAAc,CAAC,GAClE,aAAa,CAAC,2BAA2B,EAAE,UAAU,CAAC,GACtD,aAAa,CAAC,gCAAgC,EAAE,UAAU,CAAC,GAC3D,aAAa,CAAC,6CAA6C,EAAE,UAAU,CAAC,GACxE,aAAa,CAAC,0BAA0B,EAAE,UAAU,CAAC,CAAA;AAEvD,MAAM,WAAW,eAAgB,SAAQ,eAAe,CAAC,sBAAsB,CAAC;IAC9E,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,MAAM,CAAC,EAAE,WAAW,CAAA;CACrB;AAED,MAAM,WAAW,UAAU,CAAC,CAAC;IAC3B,IAAI,EAAE,MAAM,GAAG,WAAW,GAAG,QAAQ,GAAG,KAAK,GAAG,UAAU,CAAA;IAC1D,IAAI,EAAE,MAAM,CAAA;IACZ,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,GAAG,CAAA;IACR,KAAK,EAAE,MAAM,CAAA;IACb,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,CAAC,OAAO,CAAC,EAAE,eAAe,KAAK,cAAc,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,CAAA;CACzE;AAED,MAAM,WAAW,UAAW,SAAQ,UAAU,CAAC,UAAU,CAAC;IACxD,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,MAAM,CAAA;IACd,IAAI,EAAE,MAAM,CAAA;CACb;AAED,MAAM,WAAW,eAAgB,SAAQ,UAAU,CAAC,WAAW,CAAC;IAC9D,IAAI,EAAE,WAAW,CAAA;IACjB,MAAM,EAAE,MAAM,CAAA;IACd,IAAI,EAAE,MAAM,CAAA;CACb;AAED,MAAM,WAAW,UAAW,SAAQ,UAAU,CAAC,GAAG,CAAC;IACjD,IAAI,EAAE,QAAQ,CAAA;IACd,IAAI,EAAE,UAAU,CAAA;CACjB;AAED,MAAM,WAAW,OAAQ,SAAQ,UAAU,CAAC,UAAU,CAAC;IACrD,IAAI,EAAE,KAAK,CAAA;IACX,IAAI,EAAE,UAAU,CAAA;CACjB;AAED,MAAM,WAAW,YAAa,SAAQ,UAAU,CAAC,UAAU,CAAC;IAC1D,IAAI,EAAE,UAAU,CAAA;IAChB,IAAI,EAAE,UAAU,CAAA;CACjB;AAED,MAAM,MAAM,WAAW,GAAG,UAAU,GAAG,eAAe,GAAG,UAAU,GAAG,OAAO,GAAG,YAAY,CAAA;AAE5F,MAAM,WAAW,UAAU;IACzB,GAAG,EAAE,GAAG,CAAA;IACR,IAAI,EAAE,MAAM,CAAA;IACZ,IAAI,EAAE,MAAM,CAAA;IACZ,SAAS,EAAE,MAAM,EAAE,CAAA;CACpB;AAED,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,WAAW,CAAA;IAClB,IAAI,CAAC,EAAE,UAAU,CAAA;CAClB;AAED,MAAM,WAAW,OAAO;IAAG,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,eAAe,EAAE,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,aAAa,CAAC,CAAA;CAAE;AACtL,MAAM,WAAW,QAAQ;IAAG,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,eAAe,EAAE,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,aAAa,CAAC,CAAA;CAAE;AAEzM,MAAM,MAAM,mBAAmB,GAAG,aAAa,CAAC,UAAU,CAAC,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAA;AAClF,MAAM,MAAM,wBAAwB,GAAG,aAAa,CAAC,WAAW,CAAC,GAAG,QAAQ,CAAC,WAAW,CAAC,CAAA;AACzF,MAAM,MAAM,eAAe,GAAG,mBAAmB,GAAG,wBAAwB,CAAA;AAC5E,MAAM,WAAW,gBAAgB;IAAG,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,eAAe,GAAG,CAAC,OAAO,EAAE,eAAe,KAAK,eAAe,CAAA;CAAE;AAEzM,MAAM,WAAW,qBAAqB;IACpC,SAAS,EAAE,MAAM,CAAA;IACjB,UAAU,EAAE,MAAM,CAAC,OAAO,CAAC,CAAA;IAC3B,UAAU,EAAE,MAAM,CAAC,OAAO,CAAC,CAAA;CAC5B;AAED,MAAM,MAAM,eAAe,GAAG,IAAI,CAAC,UAAU,EAAE,KAAK,CAAC,CAAA;AA0CrD,wBAAwB,QAAQ,CAAE,IAAI,EAAE,MAAM,GAAG,GAAG,EAAE,UAAU,EAAE,eAAe,EAAE,OAAO,GAAE,eAAoB,GAAG,cAAc,CAAC,WAAW,EAAE,IAAI,EAAE,GAAG,CAAC,CA8BxJ;AAED,wBAAsB,QAAQ,CAAE,IAAI,EAAE,MAAM,GAAG,GAAG,EAAE,UAAU,EAAE,eAAe,EAAE,OAAO,GAAE,eAAoB,GAAG,OAAO,CAAC,WAAW,CAAC,CAQpI;AAED,wBAAwB,SAAS,CAAE,IAAI,EAAE,MAAM,GAAG,GAAG,EAAE,UAAU,EAAE,eAAe,EAAE,OAAO,GAAE,eAAoB,GAAG,cAAc,CAAC,WAAW,EAAE,IAAI,EAAE,GAAG,CAAC,CA4BzJ"}
+91
View File
@@ -0,0 +1,91 @@
import errCode from 'err-code';
import last from 'it-last';
import { CID } from 'multiformats/cid';
import resolve from './resolvers/index.js';
const toPathComponents = (path = '') => {
// split on / unless escaped with \
return (path
.trim()
.match(/([^\\^/]|\\\/)+/g) ?? [])
.filter(Boolean);
};
const cidAndRest = (path) => {
if (path instanceof Uint8Array) {
return {
cid: CID.decode(path),
toResolve: []
};
}
const cid = CID.asCID(path);
if (cid != null) {
return {
cid,
toResolve: []
};
}
if (typeof path === 'string') {
if (path.indexOf('/ipfs/') === 0) {
path = path.substring(6);
}
const output = toPathComponents(path);
return {
cid: CID.parse(output[0]),
toResolve: output.slice(1)
};
}
throw errCode(new Error(`Unknown path type ${path}`), 'ERR_BAD_PATH');
};
export async function* walkPath(path, blockstore, options = {}) {
let { cid, toResolve } = cidAndRest(path);
let name = cid.toString();
let entryPath = name;
const startingDepth = toResolve.length;
while (true) {
const result = await resolve(cid, name, entryPath, toResolve, startingDepth, blockstore, options);
if (result.entry == null && result.next == null) {
throw errCode(new Error(`Could not resolve ${path}`), 'ERR_NOT_FOUND');
}
if (result.entry != null) {
yield result.entry;
}
if (result.next == null) {
return;
}
// resolve further parts
toResolve = result.next.toResolve;
cid = result.next.cid;
name = result.next.name;
entryPath = result.next.path;
}
}
export async function exporter(path, blockstore, options = {}) {
const result = await last(walkPath(path, blockstore, options));
if (result == null) {
throw errCode(new Error(`Could not resolve ${path}`), 'ERR_NOT_FOUND');
}
return result;
}
export async function* recursive(path, blockstore, options = {}) {
const node = await exporter(path, blockstore, options);
if (node == null) {
return;
}
yield node;
if (node.type === 'directory') {
for await (const child of recurse(node, options)) {
yield child;
}
}
async function* recurse(node, options) {
for await (const file of node.content(options)) {
yield file;
if (file instanceof Uint8Array) {
continue;
}
if (file.type === 'directory') {
yield* recurse(file, options);
}
}
}
}
//# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,UAAU,CAAA;AAC9B,OAAO,IAAI,MAAM,SAAS,CAAA;AAC1B,OAAO,EAAE,GAAG,EAAE,MAAM,kBAAkB,CAAA;AACtC,OAAO,OAAO,MAAM,sBAAsB,CAAA;AAsH1C,MAAM,gBAAgB,GAAG,CAAC,OAAe,EAAE,EAAY,EAAE;IACvD,mCAAmC;IACnC,OAAO,CAAC,IAAI;SACT,IAAI,EAAE;SACN,KAAK,CAAC,kBAAkB,CAAC,IAAI,EAAE,CAAC;SAChC,MAAM,CAAC,OAAO,CAAC,CAAA;AACpB,CAAC,CAAA;AAED,MAAM,UAAU,GAAG,CAAC,IAA+B,EAAqC,EAAE;IACxF,IAAI,IAAI,YAAY,UAAU,EAAE;QAC9B,OAAO;YACL,GAAG,EAAE,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC;YACrB,SAAS,EAAE,EAAE;SACd,CAAA;KACF;IAED,MAAM,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IAC3B,IAAI,GAAG,IAAI,IAAI,EAAE;QACf,OAAO;YACL,GAAG;YACH,SAAS,EAAE,EAAE;SACd,CAAA;KACF;IAED,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE;QAC5B,IAAI,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE;YAChC,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAA;SACzB;QAED,MAAM,MAAM,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAA;QAErC,OAAO;YACL,GAAG,EAAE,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YACzB,SAAS,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;SAC3B,CAAA;KACF;IAED,MAAM,OAAO,CAAC,IAAI,KAAK,CAAC,qBAAqB,IAAI,EAAE,CAAC,EAAE,cAAc,CAAC,CAAA;AACvE,CAAC,CAAA;AAED,MAAM,CAAC,KAAK,SAAU,CAAC,CAAC,QAAQ,CAAE,IAAkB,EAAE,UAA2B,EAAE,UAA2B,EAAE;IAC9G,IAAI,EACF,GAAG,EACH,SAAS,EACV,GAAG,UAAU,CAAC,IAAI,CAAC,CAAA;IACpB,IAAI,IAAI,GAAG,GAAG,CAAC,QAAQ,EAAE,CAAA;IACzB,IAAI,SAAS,GAAG,IAAI,CAAA;IACpB,MAAM,aAAa,GAAG,SAAS,CAAC,MAAM,CAAA;IAEtC,OAAO,IAAI,EAAE;QACX,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE,IAAI,EAAE,SAAS,EAAE,SAAS,EAAE,aAAa,EAAE,UAAU,EAAE,OAAO,CAAC,CAAA;QAEjG,IAAI,MAAM,CAAC,KAAK,IAAI,IAAI,IAAI,MAAM,CAAC,IAAI,IAAI,IAAI,EAAE;YAC/C,MAAM,OAAO,CAAC,IAAI,KAAK,CAAC,qBAAqB,IAAI,EAAE,CAAC,EAAE,eAAe,CAAC,CAAA;SACvE;QAED,IAAI,MAAM,CAAC,KAAK,IAAI,IAAI,EAAE;YACxB,MAAM,MAAM,CAAC,KAAK,CAAA;SACnB;QAED,IAAI,MAAM,CAAC,IAAI,IAAI,IAAI,EAAE;YACvB,OAAM;SACP;QAED,wBAAwB;QACxB,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAA;QACjC,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAA;QACrB,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAA;QACvB,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAA;KAC7B;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAE,IAAkB,EAAE,UAA2B,EAAE,UAA2B,EAAE;IAC5G,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC,CAAA;IAE9D,IAAI,MAAM,IAAI,IAAI,EAAE;QAClB,MAAM,OAAO,CAAC,IAAI,KAAK,CAAC,qBAAqB,IAAI,EAAE,CAAC,EAAE,eAAe,CAAC,CAAA;KACvE;IAED,OAAO,MAAM,CAAA;AACf,CAAC;AAED,MAAM,CAAC,KAAK,SAAU,CAAC,CAAC,SAAS,CAAE,IAAkB,EAAE,UAA2B,EAAE,UAA2B,EAAE;IAC/G,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,UAAU,EAAE,OAAO,CAAC,CAAA;IAEtD,IAAI,IAAI,IAAI,IAAI,EAAE;QAChB,OAAM;KACP;IAED,MAAM,IAAI,CAAA;IAEV,IAAI,IAAI,CAAC,IAAI,KAAK,WAAW,EAAE;QAC7B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,EAAE;YAChD,MAAM,KAAK,CAAA;SACZ;KACF;IAED,KAAK,SAAU,CAAC,CAAC,OAAO,CAAE,IAAqB,EAAE,OAAwB;QACvE,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE;YAC9C,MAAM,IAAI,CAAA;YAEV,IAAI,IAAI,YAAY,UAAU,EAAE;gBAC9B,SAAQ;aACT;YAED,IAAI,IAAI,CAAC,IAAI,KAAK,WAAW,EAAE;gBAC7B,KAAM,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;aAC/B;SACF;IACH,CAAC;AACH,CAAC"}
@@ -0,0 +1,4 @@
import type { Resolver } from '../index.js';
declare const resolve: Resolver;
export default resolve;
//# sourceMappingURL=dag-cbor.d.ts.map
@@ -0,0 +1 @@
{"version":3,"file":"dag-cbor.d.ts","sourceRoot":"","sources":["../../../src/resolvers/dag-cbor.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AAE3C,QAAA,MAAM,OAAO,EAAE,QA2Dd,CAAA;AAED,eAAe,OAAO,CAAA"}
@@ -0,0 +1,61 @@
import * as dagCbor from '@ipld/dag-cbor';
import errCode from 'err-code';
import { CID } from 'multiformats/cid';
const resolve = async (cid, name, path, toResolve, resolve, depth, blockstore, options) => {
const block = await blockstore.get(cid, options);
const object = dagCbor.decode(block);
let subObject = object;
let subPath = path;
while (toResolve.length > 0) {
const prop = toResolve[0];
if (prop in subObject) {
// remove the bit of the path we have resolved
toResolve.shift();
subPath = `${subPath}/${prop}`;
const subObjectCid = CID.asCID(subObject[prop]);
if (subObjectCid != null) {
return {
entry: {
type: 'object',
name,
path,
cid,
node: block,
depth,
size: BigInt(block.length),
content: async function* () {
yield object;
}
},
next: {
cid: subObjectCid,
name: prop,
path: subPath,
toResolve
}
};
}
subObject = subObject[prop];
}
else {
// cannot resolve further
throw errCode(new Error(`No property named ${prop} found in cbor node ${cid}`), 'ERR_NO_PROP');
}
}
return {
entry: {
type: 'object',
name,
path,
cid,
node: block,
depth,
size: BigInt(block.length),
content: async function* () {
yield object;
}
}
};
};
export default resolve;
//# sourceMappingURL=dag-cbor.js.map
@@ -0,0 +1 @@
{"version":3,"file":"dag-cbor.js","sourceRoot":"","sources":["../../../src/resolvers/dag-cbor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,gBAAgB,CAAA;AACzC,OAAO,OAAO,MAAM,UAAU,CAAA;AAC9B,OAAO,EAAE,GAAG,EAAE,MAAM,kBAAkB,CAAA;AAGtC,MAAM,OAAO,GAAa,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,EAAE;IAClG,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;IAChD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAM,KAAK,CAAC,CAAA;IACzC,IAAI,SAAS,GAAG,MAAM,CAAA;IACtB,IAAI,OAAO,GAAG,IAAI,CAAA;IAElB,OAAO,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE;QAC3B,MAAM,IAAI,GAAG,SAAS,CAAC,CAAC,CAAC,CAAA;QAEzB,IAAI,IAAI,IAAI,SAAS,EAAE;YACrB,8CAA8C;YAC9C,SAAS,CAAC,KAAK,EAAE,CAAA;YACjB,OAAO,GAAG,GAAG,OAAO,IAAI,IAAI,EAAE,CAAA;YAE9B,MAAM,YAAY,GAAG,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAA;YAC/C,IAAI,YAAY,IAAI,IAAI,EAAE;gBACxB,OAAO;oBACL,KAAK,EAAE;wBACL,IAAI,EAAE,QAAQ;wBACd,IAAI;wBACJ,IAAI;wBACJ,GAAG;wBACH,IAAI,EAAE,KAAK;wBACX,KAAK;wBACL,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC;wBAC1B,OAAO,EAAE,KAAK,SAAU,CAAC;4BACvB,MAAM,MAAM,CAAA;wBACd,CAAC;qBACF;oBACD,IAAI,EAAE;wBACJ,GAAG,EAAE,YAAY;wBACjB,IAAI,EAAE,IAAI;wBACV,IAAI,EAAE,OAAO;wBACb,SAAS;qBACV;iBACF,CAAA;aACF;YAED,SAAS,GAAG,SAAS,CAAC,IAAI,CAAC,CAAA;SAC5B;aAAM;YACL,yBAAyB;YACzB,MAAM,OAAO,CAAC,IAAI,KAAK,CAAC,qBAAqB,IAAI,uBAAuB,GAAG,EAAE,CAAC,EAAE,aAAa,CAAC,CAAA;SAC/F;KACF;IAED,OAAO;QACL,KAAK,EAAE;YACL,IAAI,EAAE,QAAQ;YACd,IAAI;YACJ,IAAI;YACJ,GAAG;YACH,IAAI,EAAE,KAAK;YACX,KAAK;YACL,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC;YAC1B,OAAO,EAAE,KAAK,SAAU,CAAC;gBACvB,MAAM,MAAM,CAAA;YACd,CAAC;SACF;KACF,CAAA;AACH,CAAC,CAAA;AAED,eAAe,OAAO,CAAA"}
@@ -0,0 +1,4 @@
import type { Resolver } from '../index.js';
declare const resolve: Resolver;
export default resolve;
//# sourceMappingURL=identity.d.ts.map
@@ -0,0 +1 @@
{"version":3,"file":"identity.d.ts","sourceRoot":"","sources":["../../../src/resolvers/identity.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAmB,QAAQ,EAAkB,MAAM,aAAa,CAAA;AAuB5E,QAAA,MAAM,OAAO,EAAE,QAkBd,CAAA;AAED,eAAe,OAAO,CAAA"}
@@ -0,0 +1,38 @@
import errCode from 'err-code';
import * as mh from 'multiformats/hashes/digest';
import { CustomProgressEvent } from 'progress-events';
import extractDataFromBlock from '../utils/extract-data-from-block.js';
import validateOffsetAndLength from '../utils/validate-offset-and-length.js';
const rawContent = (node) => {
async function* contentGenerator(options = {}) {
const { start, end } = validateOffsetAndLength(node.length, options.offset, options.length);
const buf = extractDataFromBlock(node, 0n, start, end);
options.onProgress?.(new CustomProgressEvent('unixfs:exporter:progress:identity', {
bytesRead: BigInt(buf.byteLength),
totalBytes: end - start,
fileSize: BigInt(node.byteLength)
}));
yield buf;
}
return contentGenerator;
};
const resolve = async (cid, name, path, toResolve, resolve, depth, blockstore, options) => {
if (toResolve.length > 0) {
throw errCode(new Error(`No link named ${path} found in raw node ${cid}`), 'ERR_NOT_FOUND');
}
const buf = mh.decode(cid.multihash.bytes);
return {
entry: {
type: 'identity',
name,
path,
cid,
content: rawContent(buf.digest),
depth,
size: BigInt(buf.digest.length),
node: buf.digest
}
};
};
export default resolve;
//# sourceMappingURL=identity.js.map
@@ -0,0 +1 @@
{"version":3,"file":"identity.js","sourceRoot":"","sources":["../../../src/resolvers/identity.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,UAAU,CAAA;AAC9B,OAAO,KAAK,EAAE,MAAM,4BAA4B,CAAA;AAChD,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAA;AACrD,OAAO,oBAAoB,MAAM,qCAAqC,CAAA;AACtE,OAAO,uBAAuB,MAAM,wCAAwC,CAAA;AAG5E,MAAM,UAAU,GAAG,CAAC,IAAgB,EAAgF,EAAE;IACpH,KAAK,SAAU,CAAC,CAAC,gBAAgB,CAAE,UAA2B,EAAE;QAC9D,MAAM,EACJ,KAAK,EACL,GAAG,EACJ,GAAG,uBAAuB,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAA;QAExE,MAAM,GAAG,GAAG,oBAAoB,CAAC,IAAI,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,CAAA;QAEtD,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,mBAAmB,CAAiB,mCAAmC,EAAE;YAChG,SAAS,EAAE,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC;YACjC,UAAU,EAAE,GAAG,GAAG,KAAK;YACvB,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC;SAClC,CAAC,CAAC,CAAA;QAEH,MAAM,GAAG,CAAA;IACX,CAAC;IAED,OAAO,gBAAgB,CAAA;AACzB,CAAC,CAAA;AAED,MAAM,OAAO,GAAa,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,EAAE;IAClG,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE;QACxB,MAAM,OAAO,CAAC,IAAI,KAAK,CAAC,iBAAiB,IAAI,sBAAsB,GAAG,EAAE,CAAC,EAAE,eAAe,CAAC,CAAA;KAC5F;IACD,MAAM,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;IAE1C,OAAO;QACL,KAAK,EAAE;YACL,IAAI,EAAE,UAAU;YAChB,IAAI;YACJ,IAAI;YACJ,GAAG;YACH,OAAO,EAAE,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC;YAC/B,KAAK;YACL,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC;YAC/B,IAAI,EAAE,GAAG,CAAC,MAAM;SACjB;KACF,CAAA;AACH,CAAC,CAAA;AAED,eAAe,OAAO,CAAA"}
@@ -0,0 +1,4 @@
import type { Resolve } from '../index.js';
declare const resolve: Resolve;
export default resolve;
//# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/resolvers/index.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAE,OAAO,EAAY,MAAM,aAAa,CAAA;AASpD,QAAA,MAAM,OAAO,EAAE,OAQd,CAAA;AAED,eAAe,OAAO,CAAA"}
@@ -0,0 +1,24 @@
import * as dagCbor from '@ipld/dag-cbor';
import * as dagPb from '@ipld/dag-pb';
import errCode from 'err-code';
import * as raw from 'multiformats/codecs/raw';
import { identity } from 'multiformats/hashes/identity';
import dagCborResolver from './dag-cbor.js';
import identifyResolver from './identity.js';
import rawResolver from './raw.js';
import dagPbResolver from './unixfs-v1/index.js';
const resolvers = {
[dagPb.code]: dagPbResolver,
[raw.code]: rawResolver,
[dagCbor.code]: dagCborResolver,
[identity.code]: identifyResolver
};
const resolve = async (cid, name, path, toResolve, depth, blockstore, options) => {
const resolver = resolvers[cid.code];
if (resolver == null) {
throw errCode(new Error(`No resolver for code ${cid.code}`), 'ERR_NO_RESOLVER');
}
return resolver(cid, name, path, toResolve, resolve, depth, blockstore, options);
};
export default resolve;
//# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/resolvers/index.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,OAAO,MAAM,gBAAgB,CAAA;AACzC,OAAO,KAAK,KAAK,MAAM,cAAc,CAAA;AACrC,OAAO,OAAO,MAAM,UAAU,CAAA;AAC9B,OAAO,KAAK,GAAG,MAAM,yBAAyB,CAAA;AAC9C,OAAO,EAAE,QAAQ,EAAE,MAAM,8BAA8B,CAAA;AACvD,OAAO,eAAe,MAAM,eAAe,CAAA;AAC3C,OAAO,gBAAgB,MAAM,eAAe,CAAA;AAC5C,OAAO,WAAW,MAAM,UAAU,CAAA;AAClC,OAAO,aAAa,MAAM,sBAAsB,CAAA;AAGhD,MAAM,SAAS,GAA6B;IAC1C,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,aAAa;IAC3B,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,WAAW;IACvB,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,eAAe;IAC/B,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,gBAAgB;CAClC,CAAA;AAED,MAAM,OAAO,GAAY,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,EAAE;IACxF,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IAEpC,IAAI,QAAQ,IAAI,IAAI,EAAE;QACpB,MAAM,OAAO,CAAC,IAAI,KAAK,CAAC,wBAAwB,GAAG,CAAC,IAAI,EAAE,CAAC,EAAE,iBAAiB,CAAC,CAAA;KAChF;IAED,OAAO,QAAQ,CAAC,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,OAAO,CAAC,CAAA;AAClF,CAAC,CAAA;AAED,eAAe,OAAO,CAAA"}
@@ -0,0 +1,4 @@
import type { Resolver } from '../index.js';
declare const resolve: Resolver;
export default resolve;
//# sourceMappingURL=raw.d.ts.map
@@ -0,0 +1 @@
{"version":3,"file":"raw.d.ts","sourceRoot":"","sources":["../../../src/resolvers/raw.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAmB,QAAQ,EAAkB,MAAM,aAAa,CAAA;AAuB5E,QAAA,MAAM,OAAO,EAAE,QAmBd,CAAA;AAED,eAAe,OAAO,CAAA"}
@@ -0,0 +1,37 @@
import errCode from 'err-code';
import { CustomProgressEvent } from 'progress-events';
import extractDataFromBlock from '../utils/extract-data-from-block.js';
import validateOffsetAndLength from '../utils/validate-offset-and-length.js';
const rawContent = (node) => {
async function* contentGenerator(options = {}) {
const { start, end } = validateOffsetAndLength(node.length, options.offset, options.length);
const buf = extractDataFromBlock(node, 0n, start, end);
options.onProgress?.(new CustomProgressEvent('unixfs:exporter:progress:raw', {
bytesRead: BigInt(buf.byteLength),
totalBytes: end - start,
fileSize: BigInt(node.byteLength)
}));
yield buf;
}
return contentGenerator;
};
const resolve = async (cid, name, path, toResolve, resolve, depth, blockstore, options) => {
if (toResolve.length > 0) {
throw errCode(new Error(`No link named ${path} found in raw node ${cid}`), 'ERR_NOT_FOUND');
}
const block = await blockstore.get(cid, options);
return {
entry: {
type: 'raw',
name,
path,
cid,
content: rawContent(block),
depth,
size: BigInt(block.length),
node: block
}
};
};
export default resolve;
//# sourceMappingURL=raw.js.map
@@ -0,0 +1 @@
{"version":3,"file":"raw.js","sourceRoot":"","sources":["../../../src/resolvers/raw.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,UAAU,CAAA;AAC9B,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAA;AACrD,OAAO,oBAAoB,MAAM,qCAAqC,CAAA;AACtE,OAAO,uBAAuB,MAAM,wCAAwC,CAAA;AAG5E,MAAM,UAAU,GAAG,CAAC,IAAgB,EAAgF,EAAE;IACpH,KAAK,SAAU,CAAC,CAAC,gBAAgB,CAAE,UAA2B,EAAE;QAC9D,MAAM,EACJ,KAAK,EACL,GAAG,EACJ,GAAG,uBAAuB,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAA;QAExE,MAAM,GAAG,GAAG,oBAAoB,CAAC,IAAI,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,CAAA;QAEtD,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,mBAAmB,CAAiB,8BAA8B,EAAE;YAC3F,SAAS,EAAE,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC;YACjC,UAAU,EAAE,GAAG,GAAG,KAAK;YACvB,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC;SAClC,CAAC,CAAC,CAAA;QAEH,MAAM,GAAG,CAAA;IACX,CAAC;IAED,OAAO,gBAAgB,CAAA;AACzB,CAAC,CAAA;AAED,MAAM,OAAO,GAAa,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,EAAE;IAClG,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE;QACxB,MAAM,OAAO,CAAC,IAAI,KAAK,CAAC,iBAAiB,IAAI,sBAAsB,GAAG,EAAE,CAAC,EAAE,eAAe,CAAC,CAAA;KAC5F;IAED,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;IAEhD,OAAO;QACL,KAAK,EAAE;YACL,IAAI,EAAE,KAAK;YACX,IAAI;YACJ,IAAI;YACJ,GAAG;YACH,OAAO,EAAE,UAAU,CAAC,KAAK,CAAC;YAC1B,KAAK;YACL,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC;YAC1B,IAAI,EAAE,KAAK;SACZ;KACF,CAAA;AACH,CAAC,CAAA;AAED,eAAe,OAAO,CAAA"}
@@ -0,0 +1,4 @@
import type { UnixfsV1Resolver } from '../../../index.js';
declare const directoryContent: UnixfsV1Resolver;
export default directoryContent;
//# sourceMappingURL=directory.d.ts.map
@@ -0,0 +1 @@
{"version":3,"file":"directory.d.ts","sourceRoot":"","sources":["../../../../../src/resolvers/unixfs-v1/content/directory.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAyD,gBAAgB,EAAE,MAAM,mBAAmB,CAAA;AAEhH,QAAA,MAAM,gBAAgB,EAAE,gBA0BvB,CAAA;AAED,eAAe,gBAAgB,CAAA"}
@@ -0,0 +1,26 @@
import filter from 'it-filter';
import map from 'it-map';
import parallel from 'it-parallel';
import { pipe } from 'it-pipe';
import { CustomProgressEvent } from 'progress-events';
const directoryContent = (cid, node, unixfs, path, resolve, depth, blockstore) => {
async function* yieldDirectoryContent(options = {}) {
const offset = options.offset ?? 0;
const length = options.length ?? node.Links.length;
const links = node.Links.slice(offset, length);
options.onProgress?.(new CustomProgressEvent('unixfs:exporter:walk:directory', {
cid
}));
yield* pipe(links, source => map(source, link => {
return async () => {
const linkName = link.Name ?? '';
const linkPath = `${path}/${linkName}`;
const result = await resolve(link.Hash, linkName, linkPath, [], depth + 1, blockstore, options);
return result.entry;
};
}), source => parallel(source, { ordered: true }), source => filter(source, entry => entry != null));
}
return yieldDirectoryContent;
};
export default directoryContent;
//# sourceMappingURL=directory.js.map
@@ -0,0 +1 @@
{"version":3,"file":"directory.js","sourceRoot":"","sources":["../../../../../src/resolvers/unixfs-v1/content/directory.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,WAAW,CAAA;AAC9B,OAAO,GAAG,MAAM,QAAQ,CAAA;AACxB,OAAO,QAAQ,MAAM,aAAa,CAAA;AAClC,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAC9B,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAA;AAGrD,MAAM,gBAAgB,GAAqB,CAAC,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,EAAE;IACjG,KAAK,SAAU,CAAC,CAAC,qBAAqB,CAAE,UAA2B,EAAE;QACnE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,CAAC,CAAA;QAClC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,CAAA;QAClD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;QAE9C,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,mBAAmB,CAAa,gCAAgC,EAAE;YACzF,GAAG;SACJ,CAAC,CAAC,CAAA;QAEH,KAAM,CAAC,CAAC,IAAI,CACV,KAAK,EACL,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE;YAC3B,OAAO,KAAK,IAAI,EAAE;gBAChB,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,IAAI,EAAE,CAAA;gBAChC,MAAM,QAAQ,GAAG,GAAG,IAAI,IAAI,QAAQ,EAAE,CAAA;gBACtC,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,EAAE,EAAE,KAAK,GAAG,CAAC,EAAE,UAAU,EAAE,OAAO,CAAC,CAAA;gBAC/F,OAAO,MAAM,CAAC,KAAK,CAAA;YACrB,CAAC,CAAA;QACH,CAAC,CAAC,EACF,MAAM,CAAC,EAAE,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,EAC7C,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC,KAAK,IAAI,IAAI,CAAC,CACjD,CAAA;IACH,CAAC;IAED,OAAO,qBAAqB,CAAA;AAC9B,CAAC,CAAA;AAED,eAAe,gBAAgB,CAAA"}
@@ -0,0 +1,4 @@
import type { UnixfsV1Resolver } from '../../../index.js';
declare const fileContent: UnixfsV1Resolver;
export default fileContent;
//# sourceMappingURL=file.d.ts.map
@@ -0,0 +1 @@
{"version":3,"file":"file.d.ts","sourceRoot":"","sources":["../../../../../src/resolvers/unixfs-v1/content/file.ts"],"names":[],"mappings":"AAYA,OAAO,KAAK,EAAwC,gBAAgB,EAA+C,MAAM,mBAAmB,CAAA;AAyH5I,QAAA,MAAM,WAAW,EAAE,gBA6DlB,CAAA;AAED,eAAe,WAAW,CAAA"}
@@ -0,0 +1,152 @@
import * as dagPb from '@ipld/dag-pb';
import errCode from 'err-code';
import { UnixFS } from 'ipfs-unixfs';
import map from 'it-map';
import parallel from 'it-parallel';
import { pipe } from 'it-pipe';
import { pushable } from 'it-pushable';
import * as raw from 'multiformats/codecs/raw';
import PQueue from 'p-queue';
import { CustomProgressEvent } from 'progress-events';
import extractDataFromBlock from '../../../utils/extract-data-from-block.js';
import validateOffsetAndLength from '../../../utils/validate-offset-and-length.js';
async function walkDAG(blockstore, node, queue, streamPosition, start, end, options) {
// a `raw` node
if (node instanceof Uint8Array) {
const buf = extractDataFromBlock(node, streamPosition, start, end);
queue.push(buf);
return;
}
if (node.Data == null) {
throw errCode(new Error('no data in PBNode'), 'ERR_NOT_UNIXFS');
}
let file;
try {
file = UnixFS.unmarshal(node.Data);
}
catch (err) {
throw errCode(err, 'ERR_NOT_UNIXFS');
}
// might be a unixfs `raw` node or have data on intermediate nodes
if (file.data != null) {
const data = file.data;
const buf = extractDataFromBlock(data, streamPosition, start, end);
queue.push(buf);
streamPosition += BigInt(buf.byteLength);
}
const childOps = [];
if (node.Links.length !== file.blockSizes.length) {
throw errCode(new Error('Inconsistent block sizes and dag links'), 'ERR_NOT_UNIXFS');
}
for (let i = 0; i < node.Links.length; i++) {
const childLink = node.Links[i];
const childStart = streamPosition; // inclusive
const childEnd = childStart + file.blockSizes[i]; // exclusive
if ((start >= childStart && start < childEnd) || // child has offset byte
(end >= childStart && end <= childEnd) || // child has end byte
(start < childStart && end > childEnd)) { // child is between offset and end bytes
childOps.push({
link: childLink,
blockStart: streamPosition
});
}
streamPosition = childEnd;
if (streamPosition > end) {
break;
}
}
await pipe(childOps, (source) => map(source, (op) => {
return async () => {
const block = await blockstore.get(op.link.Hash, options);
return {
...op,
block
};
};
}), (source) => parallel(source, {
ordered: true
}), async (source) => {
for await (const { link, block, blockStart } of source) {
let child;
switch (link.Hash.code) {
case dagPb.code:
child = dagPb.decode(block);
break;
case raw.code:
child = block;
break;
default:
queue.end(errCode(new Error(`Unsupported codec: ${link.Hash.code}`), 'ERR_NOT_UNIXFS'));
return;
}
// create a queue for this child - we use a queue instead of recursion
// to avoid overflowing the stack
const childQueue = new PQueue({
concurrency: 1
});
// if any of the child jobs error, end the read queue with the error
childQueue.on('error', error => {
queue.end(error);
});
// if the job rejects the 'error' event will be emitted on the child queue
void childQueue.add(async () => {
options.onProgress?.(new CustomProgressEvent('unixfs:exporter:walk:file', {
cid: link.Hash
}));
await walkDAG(blockstore, child, queue, blockStart, start, end, options);
});
// wait for this child to complete before moving on to the next
await childQueue.onIdle();
}
});
if (streamPosition >= end) {
queue.end();
}
}
const fileContent = (cid, node, unixfs, path, resolve, depth, blockstore) => {
async function* yieldFileContent(options = {}) {
const fileSize = unixfs.fileSize();
if (fileSize === undefined) {
throw new Error('File was a directory');
}
const { start, end } = validateOffsetAndLength(fileSize, options.offset, options.length);
if (end === 0n) {
return;
}
let read = 0n;
const wanted = end - start;
const queue = pushable();
options.onProgress?.(new CustomProgressEvent('unixfs:exporter:walk:file', {
cid
}));
void walkDAG(blockstore, node, queue, 0n, start, end, options)
.catch(err => {
queue.end(err);
});
for await (const buf of queue) {
if (buf == null) {
continue;
}
read += BigInt(buf.byteLength);
if (read > wanted) {
queue.end();
throw errCode(new Error('Read too many bytes - the file size reported by the UnixFS data in the root node may be incorrect'), 'ERR_OVER_READ');
}
if (read === wanted) {
queue.end();
}
options.onProgress?.(new CustomProgressEvent('unixfs:exporter:progress:unixfs:file', {
bytesRead: read,
totalBytes: wanted,
fileSize
}));
yield buf;
}
if (read < wanted) {
throw errCode(new Error('Traversed entire DAG but did not read enough bytes'), 'ERR_UNDER_READ');
}
}
return yieldFileContent;
};
export default fileContent;
//# sourceMappingURL=file.js.map
File diff suppressed because one or more lines are too long
@@ -0,0 +1,4 @@
import type { UnixfsV1Resolver } from '../../../index.js';
declare const hamtShardedDirectoryContent: UnixfsV1Resolver;
export default hamtShardedDirectoryContent;
//# sourceMappingURL=hamt-sharded-directory.d.ts.map
@@ -0,0 +1 @@
{"version":3,"file":"hamt-sharded-directory.d.ts","sourceRoot":"","sources":["../../../../../src/resolvers/unixfs-v1/content/hamt-sharded-directory.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAsD,gBAAgB,EAA+B,MAAM,mBAAmB,CAAA;AAE1I,QAAA,MAAM,2BAA2B,EAAE,gBAUlC,CAAA;AAoCD,eAAe,2BAA2B,CAAA"}
@@ -0,0 +1,40 @@
import { decode } from '@ipld/dag-pb';
import map from 'it-map';
import parallel from 'it-parallel';
import { pipe } from 'it-pipe';
import { CustomProgressEvent } from 'progress-events';
const hamtShardedDirectoryContent = (cid, node, unixfs, path, resolve, depth, blockstore) => {
function yieldHamtDirectoryContent(options = {}) {
options.onProgress?.(new CustomProgressEvent('unixfs:exporter:walk:hamt-sharded-directory', {
cid
}));
return listDirectory(node, path, resolve, depth, blockstore, options);
}
return yieldHamtDirectoryContent;
};
async function* listDirectory(node, path, resolve, depth, blockstore, options) {
const links = node.Links;
const results = pipe(links, source => map(source, link => {
return async () => {
const name = link.Name != null ? link.Name.substring(2) : null;
if (name != null && name !== '') {
const result = await resolve(link.Hash, name, `${path}/${name}`, [], depth + 1, blockstore, options);
return { entries: result.entry == null ? [] : [result.entry] };
}
else {
// descend into subshard
const block = await blockstore.get(link.Hash, options);
node = decode(block);
options.onProgress?.(new CustomProgressEvent('unixfs:exporter:walk:hamt-sharded-directory', {
cid: link.Hash
}));
return { entries: listDirectory(node, path, resolve, depth, blockstore, options) };
}
};
}), source => parallel(source, { ordered: true }));
for await (const { entries } of results) {
yield* entries;
}
}
export default hamtShardedDirectoryContent;
//# sourceMappingURL=hamt-sharded-directory.js.map
@@ -0,0 +1 @@
{"version":3,"file":"hamt-sharded-directory.js","sourceRoot":"","sources":["../../../../../src/resolvers/unixfs-v1/content/hamt-sharded-directory.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAe,MAAM,cAAc,CAAA;AAClD,OAAO,GAAG,MAAM,QAAQ,CAAA;AACxB,OAAO,QAAQ,MAAM,aAAa,CAAA;AAClC,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAC9B,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAA;AAGrD,MAAM,2BAA2B,GAAqB,CAAC,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,EAAE;IAC5G,SAAS,yBAAyB,CAAE,UAA2B,EAAE;QAC/D,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,mBAAmB,CAAa,6CAA6C,EAAE;YACtG,GAAG;SACJ,CAAC,CAAC,CAAA;QAEH,OAAO,aAAa,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,OAAO,CAAC,CAAA;IACvE,CAAC;IAED,OAAO,yBAAyB,CAAA;AAClC,CAAC,CAAA;AAED,KAAK,SAAU,CAAC,CAAC,aAAa,CAAE,IAAY,EAAE,IAAY,EAAE,OAAgB,EAAE,KAAa,EAAE,UAA2B,EAAE,OAAwB;IAChJ,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAA;IAExB,MAAM,OAAO,GAAG,IAAI,CAClB,KAAK,EACL,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE;QAC3B,OAAO,KAAK,IAAI,EAAE;YAChB,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;YAE9D,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,KAAK,EAAE,EAAE;gBAC/B,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,GAAG,IAAI,IAAI,IAAI,EAAE,EAAE,EAAE,EAAE,KAAK,GAAG,CAAC,EAAE,UAAU,EAAE,OAAO,CAAC,CAAA;gBAEpG,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAA;aAC/D;iBAAM;gBACL,wBAAwB;gBACxB,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;gBACtD,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,CAAA;gBAEpB,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,mBAAmB,CAAa,6CAA6C,EAAE;oBACtG,GAAG,EAAE,IAAI,CAAC,IAAI;iBACf,CAAC,CAAC,CAAA;gBAEH,OAAO,EAAE,OAAO,EAAE,aAAa,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,OAAO,CAAC,EAAE,CAAA;aACnF;QACH,CAAC,CAAA;IACH,CAAC,CAAC,EACF,MAAM,CAAC,EAAE,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAC9C,CAAA;IAED,IAAI,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,OAAO,EAAE;QACvC,KAAM,CAAC,CAAC,OAAO,CAAA;KAChB;AACH,CAAC;AAED,eAAe,2BAA2B,CAAA"}
@@ -0,0 +1,4 @@
import type { UnixfsV1Resolver } from '../../../index.js';
declare const rawContent: UnixfsV1Resolver;
export default rawContent;
//# sourceMappingURL=raw.d.ts.map
@@ -0,0 +1 @@
{"version":3,"file":"raw.d.ts","sourceRoot":"","sources":["../../../../../src/resolvers/unixfs-v1/content/raw.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAA+C,gBAAgB,EAAE,MAAM,mBAAmB,CAAA;AAEtG,QAAA,MAAM,UAAU,EAAE,gBA6BjB,CAAA;AAED,eAAe,UAAU,CAAA"}
@@ -0,0 +1,25 @@
import { CustomProgressEvent } from 'progress-events';
import extractDataFromBlock from '../../../utils/extract-data-from-block.js';
import validateOffsetAndLength from '../../../utils/validate-offset-and-length.js';
const rawContent = (cid, node, unixfs, path, resolve, depth, blockstore) => {
function* yieldRawContent(options = {}) {
if (unixfs.data == null) {
throw new Error('Raw block had no data');
}
const size = unixfs.data.length;
const { start, end } = validateOffsetAndLength(size, options.offset, options.length);
options.onProgress?.(new CustomProgressEvent('unixfs:exporter:walk:raw', {
cid
}));
const buf = extractDataFromBlock(unixfs.data, 0n, start, end);
options.onProgress?.(new CustomProgressEvent('unixfs:exporter:progress:unixfs:raw', {
bytesRead: BigInt(buf.byteLength),
totalBytes: end - start,
fileSize: BigInt(unixfs.data.byteLength)
}));
yield buf;
}
return yieldRawContent;
};
export default rawContent;
//# sourceMappingURL=raw.js.map
@@ -0,0 +1 @@
{"version":3,"file":"raw.js","sourceRoot":"","sources":["../../../../../src/resolvers/unixfs-v1/content/raw.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAA;AACrD,OAAO,oBAAoB,MAAM,2CAA2C,CAAA;AAC5E,OAAO,uBAAuB,MAAM,8CAA8C,CAAA;AAGlF,MAAM,UAAU,GAAqB,CAAC,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,EAAE;IAC3F,QAAS,CAAC,CAAC,eAAe,CAAE,UAA2B,EAAE;QACvD,IAAI,MAAM,CAAC,IAAI,IAAI,IAAI,EAAE;YACvB,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAA;SACzC;QAED,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAA;QAE/B,MAAM,EACJ,KAAK,EACL,GAAG,EACJ,GAAG,uBAAuB,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAA;QAEjE,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,mBAAmB,CAAa,0BAA0B,EAAE;YACnF,GAAG;SACJ,CAAC,CAAC,CAAA;QAEH,MAAM,GAAG,GAAG,oBAAoB,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,CAAA;QAE7D,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,mBAAmB,CAAiB,qCAAqC,EAAE;YAClG,SAAS,EAAE,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC;YACjC,UAAU,EAAE,GAAG,GAAG,KAAK;YACvB,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC;SACzC,CAAC,CAAC,CAAA;QAEH,MAAM,GAAG,CAAA;IACX,CAAC;IAED,OAAO,eAAe,CAAA;AACxB,CAAC,CAAA;AAED,eAAe,UAAU,CAAA"}
@@ -0,0 +1,4 @@
import type { Resolver } from '../../index.js';
declare const unixFsResolver: Resolver;
export default unixFsResolver;
//# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/resolvers/unixfs-v1/index.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,QAAQ,EAAoB,MAAM,gBAAgB,CAAA;AAuBhE,QAAA,MAAM,cAAc,EAAE,QAwFrB,CAAA;AAED,eAAe,cAAc,CAAA"}
@@ -0,0 +1,104 @@
import { decode } from '@ipld/dag-pb';
import errCode from 'err-code';
import { UnixFS } from 'ipfs-unixfs';
import findShardCid from '../../utils/find-cid-in-shard.js';
import contentDirectory from './content/directory.js';
import contentFile from './content/file.js';
import contentHamtShardedDirectory from './content/hamt-sharded-directory.js';
const findLinkCid = (node, name) => {
const link = node.Links.find(link => link.Name === name);
return link?.Hash;
};
const contentExporters = {
raw: contentFile,
file: contentFile,
directory: contentDirectory,
'hamt-sharded-directory': contentHamtShardedDirectory,
metadata: (cid, node, unixfs, path, resolve, depth, blockstore) => {
return () => [];
},
symlink: (cid, node, unixfs, path, resolve, depth, blockstore) => {
return () => [];
}
};
// @ts-expect-error types are wrong
const unixFsResolver = async (cid, name, path, toResolve, resolve, depth, blockstore, options) => {
const block = await blockstore.get(cid, options);
const node = decode(block);
let unixfs;
let next;
if (name == null) {
name = cid.toString();
}
if (node.Data == null) {
throw errCode(new Error('no data in PBNode'), 'ERR_NOT_UNIXFS');
}
try {
unixfs = UnixFS.unmarshal(node.Data);
}
catch (err) {
// non-UnixFS dag-pb node? It could happen.
throw errCode(err, 'ERR_NOT_UNIXFS');
}
if (path == null) {
path = name;
}
if (toResolve.length > 0) {
let linkCid;
if (unixfs?.type === 'hamt-sharded-directory') {
// special case - unixfs v1 hamt shards
linkCid = await findShardCid(node, toResolve[0], blockstore);
}
else {
linkCid = findLinkCid(node, toResolve[0]);
}
if (linkCid == null) {
throw errCode(new Error('file does not exist'), 'ERR_NOT_FOUND');
}
// remove the path component we have resolved
const nextName = toResolve.shift();
const nextPath = `${path}/${nextName}`;
next = {
cid: linkCid,
toResolve,
name: nextName ?? '',
path: nextPath
};
}
const content = contentExporters[unixfs.type](cid, node, unixfs, path, resolve, depth, blockstore);
if (content == null) {
throw errCode(new Error('could not find content exporter'), 'ERR_NOT_FOUND');
}
if (unixfs.isDirectory()) {
return {
entry: {
type: 'directory',
name,
path,
cid,
content,
unixfs,
depth,
node,
size: unixfs.fileSize()
},
next
};
}
return {
entry: {
type: 'file',
name,
path,
cid,
content,
unixfs,
depth,
node,
size: unixfs.fileSize()
},
next
};
};
export default unixFsResolver;
//# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/resolvers/unixfs-v1/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAe,MAAM,cAAc,CAAA;AAClD,OAAO,OAAO,MAAM,UAAU,CAAA;AAC9B,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAA;AACpC,OAAO,YAAY,MAAM,kCAAkC,CAAA;AAC3D,OAAO,gBAAgB,MAAM,wBAAwB,CAAA;AACrD,OAAO,WAAW,MAAM,mBAAmB,CAAA;AAC3C,OAAO,2BAA2B,MAAM,qCAAqC,CAAA;AAI7E,MAAM,WAAW,GAAG,CAAC,IAAY,EAAE,IAAY,EAAmB,EAAE;IAClE,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,CAAA;IAExD,OAAO,IAAI,EAAE,IAAI,CAAA;AACnB,CAAC,CAAA;AAED,MAAM,gBAAgB,GAAqC;IACzD,GAAG,EAAE,WAAW;IAChB,IAAI,EAAE,WAAW;IACjB,SAAS,EAAE,gBAAgB;IAC3B,wBAAwB,EAAE,2BAA2B;IACrD,QAAQ,EAAE,CAAC,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,EAAE;QAChE,OAAO,GAAG,EAAE,CAAC,EAAE,CAAA;IACjB,CAAC;IACD,OAAO,EAAE,CAAC,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,EAAE;QAC/D,OAAO,GAAG,EAAE,CAAC,EAAE,CAAA;IACjB,CAAC;CACF,CAAA;AAED,mCAAmC;AACnC,MAAM,cAAc,GAAa,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,EAAE;IACzG,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;IAChD,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,CAAA;IAC1B,IAAI,MAAM,CAAA;IACV,IAAI,IAAI,CAAA;IAER,IAAI,IAAI,IAAI,IAAI,EAAE;QAChB,IAAI,GAAG,GAAG,CAAC,QAAQ,EAAE,CAAA;KACtB;IAED,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,EAAE;QACrB,MAAM,OAAO,CAAC,IAAI,KAAK,CAAC,mBAAmB,CAAC,EAAE,gBAAgB,CAAC,CAAA;KAChE;IAED,IAAI;QACF,MAAM,GAAG,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;KACrC;IAAC,OAAO,GAAQ,EAAE;QACjB,2CAA2C;QAC3C,MAAM,OAAO,CAAC,GAAG,EAAE,gBAAgB,CAAC,CAAA;KACrC;IAED,IAAI,IAAI,IAAI,IAAI,EAAE;QAChB,IAAI,GAAG,IAAI,CAAA;KACZ;IAED,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE;QACxB,IAAI,OAAO,CAAA;QAEX,IAAI,MAAM,EAAE,IAAI,KAAK,wBAAwB,EAAE;YAC7C,uCAAuC;YACvC,OAAO,GAAG,MAAM,YAAY,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,CAAA;SAC7D;aAAM;YACL,OAAO,GAAG,WAAW,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,CAAA;SAC1C;QAED,IAAI,OAAO,IAAI,IAAI,EAAE;YACnB,MAAM,OAAO,CAAC,IAAI,KAAK,CAAC,qBAAqB,CAAC,EAAE,eAAe,CAAC,CAAA;SACjE;QAED,6CAA6C;QAC7C,MAAM,QAAQ,GAAG,SAAS,CAAC,KAAK,EAAE,CAAA;QAClC,MAAM,QAAQ,GAAG,GAAG,IAAI,IAAI,QAAQ,EAAE,CAAA;QAEtC,IAAI,GAAG;YACL,GAAG,EAAE,OAAO;YACZ,SAAS;YACT,IAAI,EAAE,QAAQ,IAAI,EAAE;YACpB,IAAI,EAAE,QAAQ;SACf,CAAA;KACF;IAED,MAAM,OAAO,GAAG,gBAAgB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,CAAC,CAAA;IAElG,IAAI,OAAO,IAAI,IAAI,EAAE;QACnB,MAAM,OAAO,CAAC,IAAI,KAAK,CAAC,iCAAiC,CAAC,EAAE,eAAe,CAAC,CAAA;KAC7E;IAED,IAAI,MAAM,CAAC,WAAW,EAAE,EAAE;QACxB,OAAO;YACL,KAAK,EAAE;gBACL,IAAI,EAAE,WAAW;gBACjB,IAAI;gBACJ,IAAI;gBACJ,GAAG;gBACH,OAAO;gBACP,MAAM;gBACN,KAAK;gBACL,IAAI;gBACJ,IAAI,EAAE,MAAM,CAAC,QAAQ,EAAE;aACxB;YACD,IAAI;SACL,CAAA;KACF;IAED,OAAO;QACL,KAAK,EAAE;YACL,IAAI,EAAE,MAAM;YACZ,IAAI;YACJ,IAAI;YACJ,GAAG;YACH,OAAO;YACP,MAAM;YACN,KAAK;YACL,IAAI;YACJ,IAAI,EAAE,MAAM,CAAC,QAAQ,EAAE;SACxB;QACD,IAAI;KACL,CAAA;AACH,CAAC,CAAA;AAED,eAAe,cAAc,CAAA"}
@@ -0,0 +1,3 @@
declare function extractDataFromBlock(block: Uint8Array, blockStart: bigint, requestedStart: bigint, requestedEnd: bigint): Uint8Array;
export default extractDataFromBlock;
//# sourceMappingURL=extract-data-from-block.d.ts.map
@@ -0,0 +1 @@
{"version":3,"file":"extract-data-from-block.d.ts","sourceRoot":"","sources":["../../../src/utils/extract-data-from-block.ts"],"names":[],"mappings":"AACA,iBAAS,oBAAoB,CAAE,KAAK,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,GAAG,UAAU,CAqB9H;AAED,eAAe,oBAAoB,CAAA"}
@@ -0,0 +1,20 @@
function extractDataFromBlock(block, blockStart, requestedStart, requestedEnd) {
const blockLength = BigInt(block.length);
const blockEnd = BigInt(blockStart + blockLength);
if (requestedStart >= blockEnd || requestedEnd < blockStart) {
// If we are looking for a byte range that is starts after the start of the block,
// return an empty block. This can happen when internal nodes contain data
return new Uint8Array(0);
}
if (requestedEnd >= blockStart && requestedEnd < blockEnd) {
// If the end byte is in the current block, truncate the block to the end byte
block = block.subarray(0, Number(requestedEnd - blockStart));
}
if (requestedStart >= blockStart && requestedStart < blockEnd) {
// If the start byte is in the current block, skip to the start byte
block = block.subarray(Number(requestedStart - blockStart));
}
return block;
}
export default extractDataFromBlock;
//# sourceMappingURL=extract-data-from-block.js.map
@@ -0,0 +1 @@
{"version":3,"file":"extract-data-from-block.js","sourceRoot":"","sources":["../../../src/utils/extract-data-from-block.ts"],"names":[],"mappings":"AACA,SAAS,oBAAoB,CAAE,KAAiB,EAAE,UAAkB,EAAE,cAAsB,EAAE,YAAoB;IAChH,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;IACxC,MAAM,QAAQ,GAAG,MAAM,CAAC,UAAU,GAAG,WAAW,CAAC,CAAA;IAEjD,IAAI,cAAc,IAAI,QAAQ,IAAI,YAAY,GAAG,UAAU,EAAE;QAC3D,kFAAkF;QAClF,2EAA2E;QAC3E,OAAO,IAAI,UAAU,CAAC,CAAC,CAAC,CAAA;KACzB;IAED,IAAI,YAAY,IAAI,UAAU,IAAI,YAAY,GAAG,QAAQ,EAAE;QACzD,8EAA8E;QAC9E,KAAK,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC,YAAY,GAAG,UAAU,CAAC,CAAC,CAAA;KAC7D;IAED,IAAI,cAAc,IAAI,UAAU,IAAI,cAAc,GAAG,QAAQ,EAAE;QAC7D,oEAAoE;QACpE,KAAK,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,cAAc,GAAG,UAAU,CAAC,CAAC,CAAA;KAC5D;IAED,OAAO,KAAK,CAAA;AACd,CAAC;AAED,eAAe,oBAAoB,CAAA"}
@@ -0,0 +1,6 @@
import { type PBNode } from '@ipld/dag-pb';
import type { ExporterOptions, ShardTraversalContext, ReadableStorage } from '../index.js';
import type { CID } from 'multiformats/cid';
declare const findShardCid: (node: PBNode, name: string, blockstore: ReadableStorage, context?: ShardTraversalContext, options?: ExporterOptions) => Promise<CID | undefined>;
export default findShardCid;
//# sourceMappingURL=find-cid-in-shard.d.ts.map
@@ -0,0 +1 @@
{"version":3,"file":"find-cid-in-shard.d.ts","sourceRoot":"","sources":["../../../src/utils/find-cid-in-shard.ts"],"names":[],"mappings":"AACA,OAAO,EAAuB,KAAK,MAAM,EAAE,MAAM,cAAc,CAAA;AAG/D,OAAO,KAAK,EAAE,eAAe,EAAE,qBAAqB,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAC1F,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,kBAAkB,CAAA;AA0D3C,QAAA,MAAM,YAAY,SAAgB,MAAM,QAAQ,MAAM,cAAc,eAAe,YAAY,qBAAqB,YAAY,eAAe,KAAG,QAAQ,GAAG,GAAG,SAAS,CA4DxK,CAAA;AAED,eAAe,YAAY,CAAA"}
@@ -0,0 +1,95 @@
import { decode } from '@ipld/dag-pb';
import { murmur3128 } from '@multiformats/murmur3';
import { Bucket, createHAMT } from 'hamt-sharding';
// FIXME: this is copy/pasted from ipfs-unixfs-importer/src/options.js
const hashFn = async function (buf) {
return (await murmur3128.encode(buf))
// Murmur3 outputs 128 bit but, accidentally, IPFS Go's
// implementation only uses the first 64, so we must do the same
// for parity..
.slice(0, 8)
// Invert buffer because that's how Go impl does it
.reverse();
};
const addLinksToHamtBucket = async (links, bucket, rootBucket) => {
await Promise.all(links.map(async (link) => {
if (link.Name == null) {
// TODO(@rvagg): what do? this is technically possible
throw new Error('Unexpected Link without a Name');
}
if (link.Name.length === 2) {
const pos = parseInt(link.Name, 16);
bucket._putObjectAt(pos, new Bucket({
hash: rootBucket._options.hash,
bits: rootBucket._options.bits
}, bucket, pos));
return;
}
await rootBucket.put(link.Name.substring(2), true);
}));
};
const toPrefix = (position) => {
return position
.toString(16)
.toUpperCase()
.padStart(2, '0')
.substring(0, 2);
};
const toBucketPath = (position) => {
let bucket = position.bucket;
const path = [];
while (bucket._parent != null) {
path.push(bucket);
bucket = bucket._parent;
}
path.push(bucket);
return path.reverse();
};
const findShardCid = async (node, name, blockstore, context, options) => {
if (context == null) {
const rootBucket = createHAMT({
hashFn
});
context = {
rootBucket,
hamtDepth: 1,
lastBucket: rootBucket
};
}
await addLinksToHamtBucket(node.Links, context.lastBucket, context.rootBucket);
const position = await context.rootBucket._findNewBucketAndPos(name);
let prefix = toPrefix(position.pos);
const bucketPath = toBucketPath(position);
if (bucketPath.length > context.hamtDepth) {
context.lastBucket = bucketPath[context.hamtDepth];
prefix = toPrefix(context.lastBucket._posAtParent);
}
const link = node.Links.find(link => {
if (link.Name == null) {
return false;
}
const entryPrefix = link.Name.substring(0, 2);
const entryName = link.Name.substring(2);
if (entryPrefix !== prefix) {
// not the entry or subshard we're looking for
return false;
}
if (entryName !== '' && entryName !== name) {
// not the entry we're looking for
return false;
}
return true;
});
if (link == null) {
return;
}
if (link.Name != null && link.Name.substring(2) === name) {
return link.Hash;
}
context.hamtDepth++;
const block = await blockstore.get(link.Hash, options);
node = decode(block);
return findShardCid(node, name, blockstore, context, options);
};
export default findShardCid;
//# sourceMappingURL=find-cid-in-shard.js.map
@@ -0,0 +1 @@
{"version":3,"file":"find-cid-in-shard.js","sourceRoot":"","sources":["../../../src/utils/find-cid-in-shard.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAA4B,MAAM,cAAc,CAAA;AAC/D,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAClD,OAAO,EAAE,MAAM,EAAuB,UAAU,EAAE,MAAM,eAAe,CAAA;AAIvE,sEAAsE;AACtE,MAAM,MAAM,GAAG,KAAK,WAAW,GAAe;IAC5C,OAAO,CAAC,MAAM,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACnC,uDAAuD;QACvD,gEAAgE;QAChE,eAAe;SACd,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;QACZ,mDAAmD;SAClD,OAAO,EAAE,CAAA;AACd,CAAC,CAAA;AAED,MAAM,oBAAoB,GAAG,KAAK,EAAE,KAAe,EAAE,MAAuB,EAAE,UAA2B,EAAiB,EAAE;IAC1H,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,GAAG,CAAC,KAAK,EAAC,IAAI,EAAC,EAAE;QACrB,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,EAAE;YACrB,sDAAsD;YACtD,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAA;SAClD;QACD,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE;YAC1B,MAAM,GAAG,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC,CAAA;YAEnC,MAAM,CAAC,YAAY,CAAC,GAAG,EAAE,IAAI,MAAM,CAAC;gBAClC,IAAI,EAAE,UAAU,CAAC,QAAQ,CAAC,IAAI;gBAC9B,IAAI,EAAE,UAAU,CAAC,QAAQ,CAAC,IAAI;aAC/B,EAAE,MAAM,EAAE,GAAG,CAAC,CAAC,CAAA;YAChB,OAAM;SACP;QAED,MAAM,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAA;IACpD,CAAC,CAAC,CACH,CAAA;AACH,CAAC,CAAA;AAED,MAAM,QAAQ,GAAG,CAAC,QAAgB,EAAU,EAAE;IAC5C,OAAO,QAAQ;SACZ,QAAQ,CAAC,EAAE,CAAC;SACZ,WAAW,EAAE;SACb,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC;SAChB,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;AACpB,CAAC,CAAA;AAED,MAAM,YAAY,GAAG,CAAC,QAAiC,EAA0B,EAAE;IACjF,IAAI,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAA;IAC5B,MAAM,IAAI,GAAG,EAAE,CAAA;IAEf,OAAO,MAAM,CAAC,OAAO,IAAI,IAAI,EAAE;QAC7B,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;QAEjB,MAAM,GAAG,MAAM,CAAC,OAAO,CAAA;KACxB;IAED,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;IAEjB,OAAO,IAAI,CAAC,OAAO,EAAE,CAAA;AACvB,CAAC,CAAA;AAED,MAAM,YAAY,GAAG,KAAK,EAAE,IAAY,EAAE,IAAY,EAAE,UAA2B,EAAE,OAA+B,EAAE,OAAyB,EAA4B,EAAE;IAC3K,IAAI,OAAO,IAAI,IAAI,EAAE;QACnB,MAAM,UAAU,GAAG,UAAU,CAAU;YACrC,MAAM;SACP,CAAC,CAAA;QAEF,OAAO,GAAG;YACR,UAAU;YACV,SAAS,EAAE,CAAC;YACZ,UAAU,EAAE,UAAU;SACvB,CAAA;KACF;IAED,MAAM,oBAAoB,CAAC,IAAI,CAAC,KAAK,EAAE,OAAO,CAAC,UAAU,EAAE,OAAO,CAAC,UAAU,CAAC,CAAA;IAE9E,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,oBAAoB,CAAC,IAAI,CAAC,CAAA;IACpE,IAAI,MAAM,GAAG,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAA;IACnC,MAAM,UAAU,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAA;IAEzC,IAAI,UAAU,CAAC,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE;QACzC,OAAO,CAAC,UAAU,GAAG,UAAU,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;QAElD,MAAM,GAAG,QAAQ,CAAC,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC,CAAA;KACnD;IAED,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QAClC,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,EAAE;YACrB,OAAO,KAAK,CAAA;SACb;QAED,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;QAC7C,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAA;QAExC,IAAI,WAAW,KAAK,MAAM,EAAE;YAC1B,8CAA8C;YAC9C,OAAO,KAAK,CAAA;SACb;QAED,IAAI,SAAS,KAAK,EAAE,IAAI,SAAS,KAAK,IAAI,EAAE;YAC1C,kCAAkC;YAClC,OAAO,KAAK,CAAA;SACb;QAED,OAAO,IAAI,CAAA;IACb,CAAC,CAAC,CAAA;IAEF,IAAI,IAAI,IAAI,IAAI,EAAE;QAChB,OAAM;KACP;IAED,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE;QACxD,OAAO,IAAI,CAAC,IAAI,CAAA;KACjB;IAED,OAAO,CAAC,SAAS,EAAE,CAAA;IAEnB,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;IACtD,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,CAAA;IAEpB,OAAO,YAAY,CAAC,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,OAAO,CAAC,CAAA;AAC/D,CAAC,CAAA;AAED,eAAe,YAAY,CAAA"}
@@ -0,0 +1,6 @@
declare const validateOffsetAndLength: (size: number | bigint, offset?: number | bigint, length?: number | bigint) => {
start: bigint;
end: bigint;
};
export default validateOffsetAndLength;
//# sourceMappingURL=validate-offset-and-length.d.ts.map
@@ -0,0 +1 @@
{"version":3,"file":"validate-offset-and-length.d.ts","sourceRoot":"","sources":["../../../src/utils/validate-offset-and-length.ts"],"names":[],"mappings":"AAEA,QAAA,MAAM,uBAAuB,SAAU,MAAM,GAAG,MAAM,WAAU,MAAM,GAAG,MAAM,WAAc,MAAM,GAAG,MAAM;WAAmB,MAAM;SAAO,MAAM;CAiCjJ,CAAA;AAED,eAAe,uBAAuB,CAAA"}
@@ -0,0 +1,30 @@
import errCode from 'err-code';
const validateOffsetAndLength = (size, offset = 0, length = size) => {
const fileSize = BigInt(size);
const start = BigInt(offset ?? 0);
let end = BigInt(length);
if (end !== fileSize) {
end = start + end;
}
if (end > fileSize) {
end = fileSize;
}
if (start < 0n) {
throw errCode(new Error('Offset must be greater than or equal to 0'), 'ERR_INVALID_PARAMS');
}
if (start > fileSize) {
throw errCode(new Error('Offset must be less than the file size'), 'ERR_INVALID_PARAMS');
}
if (end < 0n) {
throw errCode(new Error('Length must be greater than or equal to 0'), 'ERR_INVALID_PARAMS');
}
if (end > fileSize) {
throw errCode(new Error('Length must be less than the file size'), 'ERR_INVALID_PARAMS');
}
return {
start,
end
};
};
export default validateOffsetAndLength;
//# sourceMappingURL=validate-offset-and-length.js.map
@@ -0,0 +1 @@
{"version":3,"file":"validate-offset-and-length.js","sourceRoot":"","sources":["../../../src/utils/validate-offset-and-length.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,UAAU,CAAA;AAE9B,MAAM,uBAAuB,GAAG,CAAC,IAAqB,EAAE,SAA0B,CAAC,EAAE,SAA0B,IAAI,EAAkC,EAAE;IACrJ,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,CAAA;IAC7B,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC,CAAA;IACjC,IAAI,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,CAAA;IAExB,IAAI,GAAG,KAAK,QAAQ,EAAE;QACpB,GAAG,GAAG,KAAK,GAAG,GAAG,CAAA;KAClB;IAED,IAAI,GAAG,GAAG,QAAQ,EAAE;QAClB,GAAG,GAAG,QAAQ,CAAA;KACf;IAED,IAAI,KAAK,GAAG,EAAE,EAAE;QACd,MAAM,OAAO,CAAC,IAAI,KAAK,CAAC,2CAA2C,CAAC,EAAE,oBAAoB,CAAC,CAAA;KAC5F;IAED,IAAI,KAAK,GAAG,QAAQ,EAAE;QACpB,MAAM,OAAO,CAAC,IAAI,KAAK,CAAC,wCAAwC,CAAC,EAAE,oBAAoB,CAAC,CAAA;KACzF;IAED,IAAI,GAAG,GAAG,EAAE,EAAE;QACZ,MAAM,OAAO,CAAC,IAAI,KAAK,CAAC,2CAA2C,CAAC,EAAE,oBAAoB,CAAC,CAAA;KAC5F;IAED,IAAI,GAAG,GAAG,QAAQ,EAAE;QAClB,MAAM,OAAO,CAAC,IAAI,KAAK,CAAC,wCAAwC,CAAC,EAAE,oBAAoB,CAAC,CAAA;KACzF;IAED,OAAO;QACL,KAAK;QACL,GAAG;KACJ,CAAA;AACH,CAAC,CAAA;AAED,eAAe,uBAAuB,CAAA"}
+26
View File
@@ -0,0 +1,26 @@
{
"ExportProgress": "https://ipfs.github.io/js-ipfs-unixfs/interfaces/ipfs_unixfs_exporter.ExportProgress.html",
"ExportWalk": "https://ipfs.github.io/js-ipfs-unixfs/interfaces/ipfs_unixfs_exporter.ExportWalk.html",
"Exportable": "https://ipfs.github.io/js-ipfs-unixfs/interfaces/ipfs_unixfs_exporter.Exportable.html",
"ExporterOptions": "https://ipfs.github.io/js-ipfs-unixfs/interfaces/ipfs_unixfs_exporter.ExporterOptions.html",
"IdentityNode": "https://ipfs.github.io/js-ipfs-unixfs/interfaces/ipfs_unixfs_exporter.IdentityNode.html",
"NextResult": "https://ipfs.github.io/js-ipfs-unixfs/interfaces/ipfs_unixfs_exporter.NextResult.html",
"ObjectNode": "https://ipfs.github.io/js-ipfs-unixfs/interfaces/ipfs_unixfs_exporter.ObjectNode.html",
"RawNode": "https://ipfs.github.io/js-ipfs-unixfs/interfaces/ipfs_unixfs_exporter.RawNode.html",
"Resolve": "https://ipfs.github.io/js-ipfs-unixfs/interfaces/ipfs_unixfs_exporter.Resolve.html",
"ResolveResult": "https://ipfs.github.io/js-ipfs-unixfs/interfaces/ipfs_unixfs_exporter.ResolveResult.html",
"Resolver": "https://ipfs.github.io/js-ipfs-unixfs/interfaces/ipfs_unixfs_exporter.Resolver.html",
"ShardTraversalContext": "https://ipfs.github.io/js-ipfs-unixfs/interfaces/ipfs_unixfs_exporter.ShardTraversalContext.html",
"UnixFSDirectory": "https://ipfs.github.io/js-ipfs-unixfs/interfaces/ipfs_unixfs_exporter.UnixFSDirectory.html",
"UnixFSFile": "https://ipfs.github.io/js-ipfs-unixfs/interfaces/ipfs_unixfs_exporter.UnixFSFile.html",
"UnixfsV1Resolver": "https://ipfs.github.io/js-ipfs-unixfs/interfaces/ipfs_unixfs_exporter.UnixfsV1Resolver.html",
"ExporterProgressEvents": "https://ipfs.github.io/js-ipfs-unixfs/types/ipfs_unixfs_exporter.ExporterProgressEvents.html",
"ReadableStorage": "https://ipfs.github.io/js-ipfs-unixfs/types/ipfs_unixfs_exporter.ReadableStorage.html",
"UnixFSEntry": "https://ipfs.github.io/js-ipfs-unixfs/types/ipfs_unixfs_exporter.UnixFSEntry.html",
"UnixfsV1Content": "https://ipfs.github.io/js-ipfs-unixfs/types/ipfs_unixfs_exporter.UnixfsV1Content.html",
"UnixfsV1DirectoryContent": "https://ipfs.github.io/js-ipfs-unixfs/types/ipfs_unixfs_exporter.UnixfsV1DirectoryContent.html",
"UnixfsV1FileContent": "https://ipfs.github.io/js-ipfs-unixfs/types/ipfs_unixfs_exporter.UnixfsV1FileContent.html",
"exporter": "https://ipfs.github.io/js-ipfs-unixfs/functions/ipfs_unixfs_exporter.exporter.html",
"recursive": "https://ipfs.github.io/js-ipfs-unixfs/functions/ipfs_unixfs_exporter.recursive.html",
"walkPath": "https://ipfs.github.io/js-ipfs-unixfs/functions/ipfs_unixfs_exporter.walkPath.html"
}
+180
View File
@@ -0,0 +1,180 @@
{
"name": "ipfs-unixfs-exporter",
"version": "13.1.5",
"description": "JavaScript implementation of the UnixFs exporter used by IPFS",
"license": "Apache-2.0 OR MIT",
"homepage": "https://github.com/ipfs/js-ipfs-unixfs/tree/master/packages/ipfs-unixfs-exporter#readme",
"repository": {
"type": "git",
"url": "git+https://github.com/ipfs/js-ipfs-unixfs.git"
},
"bugs": {
"url": "https://github.com/ipfs/js-ipfs-unixfs/issues"
},
"keywords": [
"IPFS"
],
"engines": {
"node": ">=16.0.0",
"npm": ">=7.0.0"
},
"type": "module",
"types": "./dist/src/index.d.ts",
"files": [
"src",
"dist",
"!dist/test",
"!**/*.tsbuildinfo"
],
"exports": {
".": {
"types": "./dist/src/index.d.ts",
"import": "./dist/src/index.js"
}
},
"eslintConfig": {
"extends": "ipfs",
"parserOptions": {
"sourceType": "module"
}
},
"release": {
"branches": [
"master"
],
"plugins": [
[
"@semantic-release/commit-analyzer",
{
"preset": "conventionalcommits",
"releaseRules": [
{
"breaking": true,
"release": "major"
},
{
"revert": true,
"release": "patch"
},
{
"type": "feat",
"release": "minor"
},
{
"type": "fix",
"release": "patch"
},
{
"type": "docs",
"release": "patch"
},
{
"type": "test",
"release": "patch"
},
{
"type": "deps",
"release": "patch"
},
{
"scope": "no-release",
"release": false
}
]
}
],
[
"@semantic-release/release-notes-generator",
{
"preset": "conventionalcommits",
"presetConfig": {
"types": [
{
"type": "feat",
"section": "Features"
},
{
"type": "fix",
"section": "Bug Fixes"
},
{
"type": "chore",
"section": "Trivial Changes"
},
{
"type": "docs",
"section": "Documentation"
},
{
"type": "deps",
"section": "Dependencies"
},
{
"type": "test",
"section": "Tests"
}
]
}
}
],
"@semantic-release/changelog",
"@semantic-release/npm",
"@semantic-release/github",
"@semantic-release/git"
]
},
"scripts": {
"test": "aegir test",
"test:node": "aegir test -t node --cov",
"test:chrome": "aegir test -t browser --cov",
"test:firefox": "aegir test -t browser -- --browser firefox",
"build": "aegir build",
"clean": "aegir clean",
"lint": "aegir lint",
"dep-check": "aegir dep-check",
"release": "aegir release"
},
"dependencies": {
"@ipld/dag-cbor": "^9.0.0",
"@ipld/dag-pb": "^4.0.0",
"@multiformats/murmur3": "^2.0.0",
"err-code": "^3.0.1",
"hamt-sharding": "^3.0.0",
"interface-blockstore": "^5.0.0",
"ipfs-unixfs": "^11.0.0",
"it-filter": "^3.0.2",
"it-last": "^3.0.2",
"it-map": "^3.0.3",
"it-parallel": "^3.0.0",
"it-pipe": "^3.0.1",
"it-pushable": "^3.1.0",
"multiformats": "^11.0.0",
"p-queue": "^7.3.0",
"progress-events": "^1.0.0",
"uint8arrays": "^4.0.2"
},
"devDependencies": {
"@types/readable-stream": "^2.3.15",
"@types/sinon": "^10.0.0",
"aegir": "^39.0.6",
"blockstore-core": "^4.0.1",
"delay": "^5.0.0",
"ipfs-unixfs-importer": "^15.0.0",
"iso-random-stream": "^2.0.2",
"it-all": "^3.0.2",
"it-buffer-stream": "^3.0.0",
"it-first": "^3.0.2",
"it-to-buffer": "^4.0.2",
"merge-options": "^3.0.4",
"readable-stream": "^4.4.0",
"sinon": "^15.0.0",
"wherearewe": "^2.0.1"
},
"browser": {
"fs": false,
"readable-stream": false
},
"typedoc": {
"entryPoint": "./src/index.ts"
}
}
+232
View File
@@ -0,0 +1,232 @@
import errCode from 'err-code'
import last from 'it-last'
import { CID } from 'multiformats/cid'
import resolve from './resolvers/index.js'
import type { PBNode } from '@ipld/dag-pb'
import type { Bucket } from 'hamt-sharding'
import type { Blockstore } from 'interface-blockstore'
import type { UnixFS } from 'ipfs-unixfs'
import type { ProgressOptions, ProgressEvent } from 'progress-events'
export interface ExportProgress {
/**
* How many bytes of the file have been read
*/
bytesRead: bigint
/**
* How many bytes of the file will be read - n.b. this may be
* smaller than `fileSize` if `offset`/`length` have been
* specified
*/
totalBytes: bigint
/**
* The size of the file being read - n.b. this may be
* larger than `total` if `offset`/`length` has been
* specified
*/
fileSize: bigint
}
export interface ExportWalk {
cid: CID
}
/**
* Progress events emitted by the exporter
*/
export type ExporterProgressEvents =
ProgressEvent<'unixfs:exporter:progress:unixfs:file', ExportProgress> |
ProgressEvent<'unixfs:exporter:progress:unixfs:raw', ExportProgress> |
ProgressEvent<'unixfs:exporter:progress:raw', ExportProgress> |
ProgressEvent<'unixfs:exporter:progress:identity', ExportProgress> |
ProgressEvent<'unixfs:exporter:walk:file', ExportWalk> |
ProgressEvent<'unixfs:exporter:walk:directory', ExportWalk> |
ProgressEvent<'unixfs:exporter:walk:hamt-sharded-directory', ExportWalk> |
ProgressEvent<'unixfs:exporter:walk:raw', ExportWalk>
export interface ExporterOptions extends ProgressOptions<ExporterProgressEvents> {
offset?: number
length?: number
signal?: AbortSignal
}
export interface Exportable<T> {
type: 'file' | 'directory' | 'object' | 'raw' | 'identity'
name: string
path: string
cid: CID
depth: number
size: bigint
content: (options?: ExporterOptions) => AsyncGenerator<T, void, unknown>
}
export interface UnixFSFile extends Exportable<Uint8Array> {
type: 'file'
unixfs: UnixFS
node: PBNode
}
export interface UnixFSDirectory extends Exportable<UnixFSEntry> {
type: 'directory'
unixfs: UnixFS
node: PBNode
}
export interface ObjectNode extends Exportable<any> {
type: 'object'
node: Uint8Array
}
export interface RawNode extends Exportable<Uint8Array> {
type: 'raw'
node: Uint8Array
}
export interface IdentityNode extends Exportable<Uint8Array> {
type: 'identity'
node: Uint8Array
}
export type UnixFSEntry = UnixFSFile | UnixFSDirectory | ObjectNode | RawNode | IdentityNode
export interface NextResult {
cid: CID
name: string
path: string
toResolve: string[]
}
export interface ResolveResult {
entry: UnixFSEntry
next?: NextResult
}
export interface Resolve { (cid: CID, name: string, path: string, toResolve: string[], depth: number, blockstore: ReadableStorage, options: ExporterOptions): Promise<ResolveResult> }
export interface Resolver { (cid: CID, name: string, path: string, toResolve: string[], resolve: Resolve, depth: number, blockstore: ReadableStorage, options: ExporterOptions): Promise<ResolveResult> }
export type UnixfsV1FileContent = AsyncIterable<Uint8Array> | Iterable<Uint8Array>
export type UnixfsV1DirectoryContent = AsyncIterable<UnixFSEntry> | Iterable<UnixFSEntry>
export type UnixfsV1Content = UnixfsV1FileContent | UnixfsV1DirectoryContent
export interface UnixfsV1Resolver { (cid: CID, node: PBNode, unixfs: UnixFS, path: string, resolve: Resolve, depth: number, blockstore: ReadableStorage): (options: ExporterOptions) => UnixfsV1Content }
export interface ShardTraversalContext {
hamtDepth: number
rootBucket: Bucket<boolean>
lastBucket: Bucket<boolean>
}
export type ReadableStorage = Pick<Blockstore, 'get'>
const toPathComponents = (path: string = ''): string[] => {
// split on / unless escaped with \
return (path
.trim()
.match(/([^\\^/]|\\\/)+/g) ?? [])
.filter(Boolean)
}
const cidAndRest = (path: string | Uint8Array | CID): { cid: CID, toResolve: string[] } => {
if (path instanceof Uint8Array) {
return {
cid: CID.decode(path),
toResolve: []
}
}
const cid = CID.asCID(path)
if (cid != null) {
return {
cid,
toResolve: []
}
}
if (typeof path === 'string') {
if (path.indexOf('/ipfs/') === 0) {
path = path.substring(6)
}
const output = toPathComponents(path)
return {
cid: CID.parse(output[0]),
toResolve: output.slice(1)
}
}
throw errCode(new Error(`Unknown path type ${path}`), 'ERR_BAD_PATH')
}
export async function * walkPath (path: string | CID, blockstore: ReadableStorage, options: ExporterOptions = {}): AsyncGenerator<UnixFSEntry, void, any> {
let {
cid,
toResolve
} = cidAndRest(path)
let name = cid.toString()
let entryPath = name
const startingDepth = toResolve.length
while (true) {
const result = await resolve(cid, name, entryPath, toResolve, startingDepth, blockstore, options)
if (result.entry == null && result.next == null) {
throw errCode(new Error(`Could not resolve ${path}`), 'ERR_NOT_FOUND')
}
if (result.entry != null) {
yield result.entry
}
if (result.next == null) {
return
}
// resolve further parts
toResolve = result.next.toResolve
cid = result.next.cid
name = result.next.name
entryPath = result.next.path
}
}
export async function exporter (path: string | CID, blockstore: ReadableStorage, options: ExporterOptions = {}): Promise<UnixFSEntry> {
const result = await last(walkPath(path, blockstore, options))
if (result == null) {
throw errCode(new Error(`Could not resolve ${path}`), 'ERR_NOT_FOUND')
}
return result
}
export async function * recursive (path: string | CID, blockstore: ReadableStorage, options: ExporterOptions = {}): AsyncGenerator<UnixFSEntry, void, any> {
const node = await exporter(path, blockstore, options)
if (node == null) {
return
}
yield node
if (node.type === 'directory') {
for await (const child of recurse(node, options)) {
yield child
}
}
async function * recurse (node: UnixFSDirectory, options: ExporterOptions): AsyncGenerator<UnixFSEntry, void, any> {
for await (const file of node.content(options)) {
yield file
if (file instanceof Uint8Array) {
continue
}
if (file.type === 'directory') {
yield * recurse(file, options)
}
}
}
}
@@ -0,0 +1,67 @@
import * as dagCbor from '@ipld/dag-cbor'
import errCode from 'err-code'
import { CID } from 'multiformats/cid'
import type { Resolver } from '../index.js'
const resolve: Resolver = async (cid, name, path, toResolve, resolve, depth, blockstore, options) => {
const block = await blockstore.get(cid, options)
const object = dagCbor.decode<any>(block)
let subObject = object
let subPath = path
while (toResolve.length > 0) {
const prop = toResolve[0]
if (prop in subObject) {
// remove the bit of the path we have resolved
toResolve.shift()
subPath = `${subPath}/${prop}`
const subObjectCid = CID.asCID(subObject[prop])
if (subObjectCid != null) {
return {
entry: {
type: 'object',
name,
path,
cid,
node: block,
depth,
size: BigInt(block.length),
content: async function * () {
yield object
}
},
next: {
cid: subObjectCid,
name: prop,
path: subPath,
toResolve
}
}
}
subObject = subObject[prop]
} else {
// cannot resolve further
throw errCode(new Error(`No property named ${prop} found in cbor node ${cid}`), 'ERR_NO_PROP')
}
}
return {
entry: {
type: 'object',
name,
path,
cid,
node: block,
depth,
size: BigInt(block.length),
content: async function * () {
yield object
}
}
}
}
export default resolve
@@ -0,0 +1,49 @@
import errCode from 'err-code'
import * as mh from 'multiformats/hashes/digest'
import { CustomProgressEvent } from 'progress-events'
import extractDataFromBlock from '../utils/extract-data-from-block.js'
import validateOffsetAndLength from '../utils/validate-offset-and-length.js'
import type { ExporterOptions, Resolver, ExportProgress } from '../index.js'
const rawContent = (node: Uint8Array): ((options?: ExporterOptions) => AsyncGenerator<Uint8Array, void, undefined>) => {
async function * contentGenerator (options: ExporterOptions = {}): AsyncGenerator<Uint8Array, void, undefined> {
const {
start,
end
} = validateOffsetAndLength(node.length, options.offset, options.length)
const buf = extractDataFromBlock(node, 0n, start, end)
options.onProgress?.(new CustomProgressEvent<ExportProgress>('unixfs:exporter:progress:identity', {
bytesRead: BigInt(buf.byteLength),
totalBytes: end - start,
fileSize: BigInt(node.byteLength)
}))
yield buf
}
return contentGenerator
}
const resolve: Resolver = async (cid, name, path, toResolve, resolve, depth, blockstore, options) => {
if (toResolve.length > 0) {
throw errCode(new Error(`No link named ${path} found in raw node ${cid}`), 'ERR_NOT_FOUND')
}
const buf = mh.decode(cid.multihash.bytes)
return {
entry: {
type: 'identity',
name,
path,
cid,
content: rawContent(buf.digest),
depth,
size: BigInt(buf.digest.length),
node: buf.digest
}
}
}
export default resolve
+30
View File
@@ -0,0 +1,30 @@
import * as dagCbor from '@ipld/dag-cbor'
import * as dagPb from '@ipld/dag-pb'
import errCode from 'err-code'
import * as raw from 'multiformats/codecs/raw'
import { identity } from 'multiformats/hashes/identity'
import dagCborResolver from './dag-cbor.js'
import identifyResolver from './identity.js'
import rawResolver from './raw.js'
import dagPbResolver from './unixfs-v1/index.js'
import type { Resolve, Resolver } from '../index.js'
const resolvers: Record<number, Resolver> = {
[dagPb.code]: dagPbResolver,
[raw.code]: rawResolver,
[dagCbor.code]: dagCborResolver,
[identity.code]: identifyResolver
}
const resolve: Resolve = async (cid, name, path, toResolve, depth, blockstore, options) => {
const resolver = resolvers[cid.code]
if (resolver == null) {
throw errCode(new Error(`No resolver for code ${cid.code}`), 'ERR_NO_RESOLVER')
}
return resolver(cid, name, path, toResolve, resolve, depth, blockstore, options)
}
export default resolve
+49
View File
@@ -0,0 +1,49 @@
import errCode from 'err-code'
import { CustomProgressEvent } from 'progress-events'
import extractDataFromBlock from '../utils/extract-data-from-block.js'
import validateOffsetAndLength from '../utils/validate-offset-and-length.js'
import type { ExporterOptions, Resolver, ExportProgress } from '../index.js'
const rawContent = (node: Uint8Array): ((options?: ExporterOptions) => AsyncGenerator<Uint8Array, void, undefined>) => {
async function * contentGenerator (options: ExporterOptions = {}): AsyncGenerator<Uint8Array, void, undefined> {
const {
start,
end
} = validateOffsetAndLength(node.length, options.offset, options.length)
const buf = extractDataFromBlock(node, 0n, start, end)
options.onProgress?.(new CustomProgressEvent<ExportProgress>('unixfs:exporter:progress:raw', {
bytesRead: BigInt(buf.byteLength),
totalBytes: end - start,
fileSize: BigInt(node.byteLength)
}))
yield buf
}
return contentGenerator
}
const resolve: Resolver = async (cid, name, path, toResolve, resolve, depth, blockstore, options) => {
if (toResolve.length > 0) {
throw errCode(new Error(`No link named ${path} found in raw node ${cid}`), 'ERR_NOT_FOUND')
}
const block = await blockstore.get(cid, options)
return {
entry: {
type: 'raw',
name,
path,
cid,
content: rawContent(block),
depth,
size: BigInt(block.length),
node: block
}
}
}
export default resolve
@@ -0,0 +1,36 @@
import filter from 'it-filter'
import map from 'it-map'
import parallel from 'it-parallel'
import { pipe } from 'it-pipe'
import { CustomProgressEvent } from 'progress-events'
import type { ExporterOptions, ExportWalk, UnixfsV1DirectoryContent, UnixfsV1Resolver } from '../../../index.js'
const directoryContent: UnixfsV1Resolver = (cid, node, unixfs, path, resolve, depth, blockstore) => {
async function * yieldDirectoryContent (options: ExporterOptions = {}): UnixfsV1DirectoryContent {
const offset = options.offset ?? 0
const length = options.length ?? node.Links.length
const links = node.Links.slice(offset, length)
options.onProgress?.(new CustomProgressEvent<ExportWalk>('unixfs:exporter:walk:directory', {
cid
}))
yield * pipe(
links,
source => map(source, link => {
return async () => {
const linkName = link.Name ?? ''
const linkPath = `${path}/${linkName}`
const result = await resolve(link.Hash, linkName, linkPath, [], depth + 1, blockstore, options)
return result.entry
}
}),
source => parallel(source, { ordered: true }),
source => filter(source, entry => entry != null)
)
}
return yieldDirectoryContent
}
export default directoryContent
@@ -0,0 +1,197 @@
import * as dagPb from '@ipld/dag-pb'
import errCode from 'err-code'
import { UnixFS } from 'ipfs-unixfs'
import map from 'it-map'
import parallel from 'it-parallel'
import { pipe } from 'it-pipe'
import { type Pushable, pushable } from 'it-pushable'
import * as raw from 'multiformats/codecs/raw'
import PQueue from 'p-queue'
import { CustomProgressEvent } from 'progress-events'
import extractDataFromBlock from '../../../utils/extract-data-from-block.js'
import validateOffsetAndLength from '../../../utils/validate-offset-and-length.js'
import type { ExporterOptions, UnixfsV1FileContent, UnixfsV1Resolver, ReadableStorage, ExportProgress, ExportWalk } from '../../../index.js'
async function walkDAG (blockstore: ReadableStorage, node: dagPb.PBNode | Uint8Array, queue: Pushable<Uint8Array>, streamPosition: bigint, start: bigint, end: bigint, options: ExporterOptions): Promise<void> {
// a `raw` node
if (node instanceof Uint8Array) {
const buf = extractDataFromBlock(node, streamPosition, start, end)
queue.push(buf)
return
}
if (node.Data == null) {
throw errCode(new Error('no data in PBNode'), 'ERR_NOT_UNIXFS')
}
let file: UnixFS
try {
file = UnixFS.unmarshal(node.Data)
} catch (err: any) {
throw errCode(err, 'ERR_NOT_UNIXFS')
}
// might be a unixfs `raw` node or have data on intermediate nodes
if (file.data != null) {
const data = file.data
const buf = extractDataFromBlock(data, streamPosition, start, end)
queue.push(buf)
streamPosition += BigInt(buf.byteLength)
}
const childOps: Array<{ link: dagPb.PBLink, blockStart: bigint }> = []
if (node.Links.length !== file.blockSizes.length) {
throw errCode(new Error('Inconsistent block sizes and dag links'), 'ERR_NOT_UNIXFS')
}
for (let i = 0; i < node.Links.length; i++) {
const childLink = node.Links[i]
const childStart = streamPosition // inclusive
const childEnd = childStart + file.blockSizes[i] // exclusive
if ((start >= childStart && start < childEnd) || // child has offset byte
(end >= childStart && end <= childEnd) || // child has end byte
(start < childStart && end > childEnd)) { // child is between offset and end bytes
childOps.push({
link: childLink,
blockStart: streamPosition
})
}
streamPosition = childEnd
if (streamPosition > end) {
break
}
}
await pipe(
childOps,
(source) => map(source, (op) => {
return async () => {
const block = await blockstore.get(op.link.Hash, options)
return {
...op,
block
}
}
}),
(source) => parallel(source, {
ordered: true
}),
async (source) => {
for await (const { link, block, blockStart } of source) {
let child: dagPb.PBNode | Uint8Array
switch (link.Hash.code) {
case dagPb.code:
child = dagPb.decode(block)
break
case raw.code:
child = block
break
default:
queue.end(errCode(new Error(`Unsupported codec: ${link.Hash.code}`), 'ERR_NOT_UNIXFS'))
return
}
// create a queue for this child - we use a queue instead of recursion
// to avoid overflowing the stack
const childQueue = new PQueue({
concurrency: 1
})
// if any of the child jobs error, end the read queue with the error
childQueue.on('error', error => {
queue.end(error)
})
// if the job rejects the 'error' event will be emitted on the child queue
void childQueue.add(async () => {
options.onProgress?.(new CustomProgressEvent<ExportWalk>('unixfs:exporter:walk:file', {
cid: link.Hash
}))
await walkDAG(blockstore, child, queue, blockStart, start, end, options)
})
// wait for this child to complete before moving on to the next
await childQueue.onIdle()
}
}
)
if (streamPosition >= end) {
queue.end()
}
}
const fileContent: UnixfsV1Resolver = (cid, node, unixfs, path, resolve, depth, blockstore) => {
async function * yieldFileContent (options: ExporterOptions = {}): UnixfsV1FileContent {
const fileSize = unixfs.fileSize()
if (fileSize === undefined) {
throw new Error('File was a directory')
}
const {
start,
end
} = validateOffsetAndLength(fileSize, options.offset, options.length)
if (end === 0n) {
return
}
let read = 0n
const wanted = end - start
const queue = pushable()
options.onProgress?.(new CustomProgressEvent<ExportWalk>('unixfs:exporter:walk:file', {
cid
}))
void walkDAG(blockstore, node, queue, 0n, start, end, options)
.catch(err => {
queue.end(err)
})
for await (const buf of queue) {
if (buf == null) {
continue
}
read += BigInt(buf.byteLength)
if (read > wanted) {
queue.end()
throw errCode(new Error('Read too many bytes - the file size reported by the UnixFS data in the root node may be incorrect'), 'ERR_OVER_READ')
}
if (read === wanted) {
queue.end()
}
options.onProgress?.(new CustomProgressEvent<ExportProgress>('unixfs:exporter:progress:unixfs:file', {
bytesRead: read,
totalBytes: wanted,
fileSize
}))
yield buf
}
if (read < wanted) {
throw errCode(new Error('Traversed entire DAG but did not read enough bytes'), 'ERR_UNDER_READ')
}
}
return yieldFileContent
}
export default fileContent
@@ -0,0 +1,54 @@
import { decode, type PBNode } from '@ipld/dag-pb'
import map from 'it-map'
import parallel from 'it-parallel'
import { pipe } from 'it-pipe'
import { CustomProgressEvent } from 'progress-events'
import type { ExporterOptions, Resolve, UnixfsV1DirectoryContent, UnixfsV1Resolver, ReadableStorage, ExportWalk } from '../../../index.js'
const hamtShardedDirectoryContent: UnixfsV1Resolver = (cid, node, unixfs, path, resolve, depth, blockstore) => {
function yieldHamtDirectoryContent (options: ExporterOptions = {}): UnixfsV1DirectoryContent {
options.onProgress?.(new CustomProgressEvent<ExportWalk>('unixfs:exporter:walk:hamt-sharded-directory', {
cid
}))
return listDirectory(node, path, resolve, depth, blockstore, options)
}
return yieldHamtDirectoryContent
}
async function * listDirectory (node: PBNode, path: string, resolve: Resolve, depth: number, blockstore: ReadableStorage, options: ExporterOptions): UnixfsV1DirectoryContent {
const links = node.Links
const results = pipe(
links,
source => map(source, link => {
return async () => {
const name = link.Name != null ? link.Name.substring(2) : null
if (name != null && name !== '') {
const result = await resolve(link.Hash, name, `${path}/${name}`, [], depth + 1, blockstore, options)
return { entries: result.entry == null ? [] : [result.entry] }
} else {
// descend into subshard
const block = await blockstore.get(link.Hash, options)
node = decode(block)
options.onProgress?.(new CustomProgressEvent<ExportWalk>('unixfs:exporter:walk:hamt-sharded-directory', {
cid: link.Hash
}))
return { entries: listDirectory(node, path, resolve, depth, blockstore, options) }
}
}
}),
source => parallel(source, { ordered: true })
)
for await (const { entries } of results) {
yield * entries
}
}
export default hamtShardedDirectoryContent
@@ -0,0 +1,37 @@
import { CustomProgressEvent } from 'progress-events'
import extractDataFromBlock from '../../../utils/extract-data-from-block.js'
import validateOffsetAndLength from '../../../utils/validate-offset-and-length.js'
import type { ExporterOptions, ExportProgress, ExportWalk, UnixfsV1Resolver } from '../../../index.js'
const rawContent: UnixfsV1Resolver = (cid, node, unixfs, path, resolve, depth, blockstore) => {
function * yieldRawContent (options: ExporterOptions = {}): Generator<Uint8Array, void, undefined> {
if (unixfs.data == null) {
throw new Error('Raw block had no data')
}
const size = unixfs.data.length
const {
start,
end
} = validateOffsetAndLength(size, options.offset, options.length)
options.onProgress?.(new CustomProgressEvent<ExportWalk>('unixfs:exporter:walk:raw', {
cid
}))
const buf = extractDataFromBlock(unixfs.data, 0n, start, end)
options.onProgress?.(new CustomProgressEvent<ExportProgress>('unixfs:exporter:progress:unixfs:raw', {
bytesRead: BigInt(buf.byteLength),
totalBytes: end - start,
fileSize: BigInt(unixfs.data.byteLength)
}))
yield buf
}
return yieldRawContent
}
export default rawContent
@@ -0,0 +1,121 @@
import { decode, type PBNode } from '@ipld/dag-pb'
import errCode from 'err-code'
import { UnixFS } from 'ipfs-unixfs'
import findShardCid from '../../utils/find-cid-in-shard.js'
import contentDirectory from './content/directory.js'
import contentFile from './content/file.js'
import contentHamtShardedDirectory from './content/hamt-sharded-directory.js'
import type { Resolver, UnixfsV1Resolver } from '../../index.js'
import type { CID } from 'multiformats/cid'
const findLinkCid = (node: PBNode, name: string): CID | undefined => {
const link = node.Links.find(link => link.Name === name)
return link?.Hash
}
const contentExporters: Record<string, UnixfsV1Resolver> = {
raw: contentFile,
file: contentFile,
directory: contentDirectory,
'hamt-sharded-directory': contentHamtShardedDirectory,
metadata: (cid, node, unixfs, path, resolve, depth, blockstore) => {
return () => []
},
symlink: (cid, node, unixfs, path, resolve, depth, blockstore) => {
return () => []
}
}
// @ts-expect-error types are wrong
const unixFsResolver: Resolver = async (cid, name, path, toResolve, resolve, depth, blockstore, options) => {
const block = await blockstore.get(cid, options)
const node = decode(block)
let unixfs
let next
if (name == null) {
name = cid.toString()
}
if (node.Data == null) {
throw errCode(new Error('no data in PBNode'), 'ERR_NOT_UNIXFS')
}
try {
unixfs = UnixFS.unmarshal(node.Data)
} catch (err: any) {
// non-UnixFS dag-pb node? It could happen.
throw errCode(err, 'ERR_NOT_UNIXFS')
}
if (path == null) {
path = name
}
if (toResolve.length > 0) {
let linkCid
if (unixfs?.type === 'hamt-sharded-directory') {
// special case - unixfs v1 hamt shards
linkCid = await findShardCid(node, toResolve[0], blockstore)
} else {
linkCid = findLinkCid(node, toResolve[0])
}
if (linkCid == null) {
throw errCode(new Error('file does not exist'), 'ERR_NOT_FOUND')
}
// remove the path component we have resolved
const nextName = toResolve.shift()
const nextPath = `${path}/${nextName}`
next = {
cid: linkCid,
toResolve,
name: nextName ?? '',
path: nextPath
}
}
const content = contentExporters[unixfs.type](cid, node, unixfs, path, resolve, depth, blockstore)
if (content == null) {
throw errCode(new Error('could not find content exporter'), 'ERR_NOT_FOUND')
}
if (unixfs.isDirectory()) {
return {
entry: {
type: 'directory',
name,
path,
cid,
content,
unixfs,
depth,
node,
size: unixfs.fileSize()
},
next
}
}
return {
entry: {
type: 'file',
name,
path,
cid,
content,
unixfs,
depth,
node,
size: unixfs.fileSize()
},
next
}
}
export default unixFsResolver
@@ -0,0 +1,25 @@
function extractDataFromBlock (block: Uint8Array, blockStart: bigint, requestedStart: bigint, requestedEnd: bigint): Uint8Array {
const blockLength = BigInt(block.length)
const blockEnd = BigInt(blockStart + blockLength)
if (requestedStart >= blockEnd || requestedEnd < blockStart) {
// If we are looking for a byte range that is starts after the start of the block,
// return an empty block. This can happen when internal nodes contain data
return new Uint8Array(0)
}
if (requestedEnd >= blockStart && requestedEnd < blockEnd) {
// If the end byte is in the current block, truncate the block to the end byte
block = block.subarray(0, Number(requestedEnd - blockStart))
}
if (requestedStart >= blockStart && requestedStart < blockEnd) {
// If the start byte is in the current block, skip to the start byte
block = block.subarray(Number(requestedStart - blockStart))
}
return block
}
export default extractDataFromBlock
@@ -0,0 +1,126 @@
import { decode, type PBLink, type PBNode } from '@ipld/dag-pb'
import { murmur3128 } from '@multiformats/murmur3'
import { Bucket, type BucketPosition, createHAMT } from 'hamt-sharding'
import type { ExporterOptions, ShardTraversalContext, ReadableStorage } from '../index.js'
import type { CID } from 'multiformats/cid'
// FIXME: this is copy/pasted from ipfs-unixfs-importer/src/options.js
const hashFn = async function (buf: Uint8Array): Promise<Uint8Array> {
return (await murmur3128.encode(buf))
// Murmur3 outputs 128 bit but, accidentally, IPFS Go's
// implementation only uses the first 64, so we must do the same
// for parity..
.slice(0, 8)
// Invert buffer because that's how Go impl does it
.reverse()
}
const addLinksToHamtBucket = async (links: PBLink[], bucket: Bucket<boolean>, rootBucket: Bucket<boolean>): Promise<void> => {
await Promise.all(
links.map(async link => {
if (link.Name == null) {
// TODO(@rvagg): what do? this is technically possible
throw new Error('Unexpected Link without a Name')
}
if (link.Name.length === 2) {
const pos = parseInt(link.Name, 16)
bucket._putObjectAt(pos, new Bucket({
hash: rootBucket._options.hash,
bits: rootBucket._options.bits
}, bucket, pos))
return
}
await rootBucket.put(link.Name.substring(2), true)
})
)
}
const toPrefix = (position: number): string => {
return position
.toString(16)
.toUpperCase()
.padStart(2, '0')
.substring(0, 2)
}
const toBucketPath = (position: BucketPosition<boolean>): Array<Bucket<boolean>> => {
let bucket = position.bucket
const path = []
while (bucket._parent != null) {
path.push(bucket)
bucket = bucket._parent
}
path.push(bucket)
return path.reverse()
}
const findShardCid = async (node: PBNode, name: string, blockstore: ReadableStorage, context?: ShardTraversalContext, options?: ExporterOptions): Promise<CID | undefined> => {
if (context == null) {
const rootBucket = createHAMT<boolean>({
hashFn
})
context = {
rootBucket,
hamtDepth: 1,
lastBucket: rootBucket
}
}
await addLinksToHamtBucket(node.Links, context.lastBucket, context.rootBucket)
const position = await context.rootBucket._findNewBucketAndPos(name)
let prefix = toPrefix(position.pos)
const bucketPath = toBucketPath(position)
if (bucketPath.length > context.hamtDepth) {
context.lastBucket = bucketPath[context.hamtDepth]
prefix = toPrefix(context.lastBucket._posAtParent)
}
const link = node.Links.find(link => {
if (link.Name == null) {
return false
}
const entryPrefix = link.Name.substring(0, 2)
const entryName = link.Name.substring(2)
if (entryPrefix !== prefix) {
// not the entry or subshard we're looking for
return false
}
if (entryName !== '' && entryName !== name) {
// not the entry we're looking for
return false
}
return true
})
if (link == null) {
return
}
if (link.Name != null && link.Name.substring(2) === name) {
return link.Hash
}
context.hamtDepth++
const block = await blockstore.get(link.Hash, options)
node = decode(block)
return findShardCid(node, name, blockstore, context, options)
}
export default findShardCid
@@ -0,0 +1,38 @@
import errCode from 'err-code'
const validateOffsetAndLength = (size: number | bigint, offset: number | bigint = 0, length: number | bigint = size): { start: bigint, end: bigint } => {
const fileSize = BigInt(size)
const start = BigInt(offset ?? 0)
let end = BigInt(length)
if (end !== fileSize) {
end = start + end
}
if (end > fileSize) {
end = fileSize
}
if (start < 0n) {
throw errCode(new Error('Offset must be greater than or equal to 0'), 'ERR_INVALID_PARAMS')
}
if (start > fileSize) {
throw errCode(new Error('Offset must be less than the file size'), 'ERR_INVALID_PARAMS')
}
if (end < 0n) {
throw errCode(new Error('Length must be greater than or equal to 0'), 'ERR_INVALID_PARAMS')
}
if (end > fileSize) {
throw errCode(new Error('Length must be less than the file size'), 'ERR_INVALID_PARAMS')
}
return {
start,
end
}
}
export default validateOffsetAndLength