From dc92169a996566f581bfaffb0a11f07e53818e38 Mon Sep 17 00:00:00 2001 From: Benjamin Reigner Date: Wed, 4 May 2022 13:05:53 +0200 Subject: [PATCH 1/4] feat: handle signals In order for the program to close somewhat cleanly, it now catches the signal and deletes the temporary file and folder before quitting. Also the temporary files have been renamed to have meaningful file names --- src/blockstore/fs.ts | 5 ++-- src/pack/fs.ts | 9 +++---- src/pack/stream.ts | 57 +++++++++++++++++++++++++++++++------------- src/unpack/fs.ts | 4 ++-- 4 files changed, 50 insertions(+), 25 deletions(-) diff --git a/src/blockstore/fs.ts b/src/blockstore/fs.ts index ed9106e..831e110 100644 --- a/src/blockstore/fs.ts +++ b/src/blockstore/fs.ts @@ -1,5 +1,4 @@ import fs from 'fs' -import os from 'os' import { CID } from 'multiformats' import { BaseBlockstore } from 'blockstore-core' @@ -10,9 +9,9 @@ export class FsBlockStore extends BaseBlockstore implements Blockstore { _opened: boolean _opening?: Promise - constructor () { + constructor (path: string) { super() - this.path = `${os.tmpdir()}/${(parseInt(String(Math.random() * 1e9), 10)).toString() + Date.now()}` + this.path = path this._opened = false } diff --git a/src/pack/fs.ts b/src/pack/fs.ts index 6dcf51c..fed3990 100644 --- a/src/pack/fs.ts +++ b/src/pack/fs.ts @@ -1,5 +1,4 @@ import fs from 'fs' -import os from 'os' import path from 'path' import moveFile from 'move-file' @@ -13,8 +12,10 @@ export interface PackToFsProperties extends PackProperties { } export async function packToFs ({ input, output, blockstore: userBlockstore, hasher, maxChunkSize, maxChildrenPerNode, wrapWithDirectory, rawLeaves }: PackToFsProperties) { - const blockstore = userBlockstore ? userBlockstore : new FsBlockStore() - const location = output || `${os.tmpdir()}/${(parseInt(String(Math.random() * 1e9), 10)).toString() + Date.now()}` + const realpath = path.basename(await fs.promises.realpath(input as string)) + const inputBasename = realpath === "/" ? "file" : realpath + const blockstore = userBlockstore ? userBlockstore : new FsBlockStore(`/tmp/${inputBasename}.tmp.${process.pid}`) + const location = output || `${process.cwd()}/.${inputBasename}.car.tmp.${process.pid}` const writable = fs.createWriteStream(location) const { root } = await packToStream({ @@ -35,7 +36,7 @@ export async function packToFs ({ input, output, blockstore: userBlockstore, has // Move to work dir if (!output) { const basename = typeof input === 'string' ? path.parse(path.basename(input)).name : root.toString() - const filename = `${basename}.car` + const filename = basename === "/" ? "file.car" : `${basename}.car` await moveFile(location, `${process.cwd()}/${filename}`) return {root, filename} diff --git a/src/pack/stream.ts b/src/pack/stream.ts index 8306bb1..d3c559c 100644 --- a/src/pack/stream.ts +++ b/src/pack/stream.ts @@ -14,6 +14,8 @@ import { MemoryBlockStore } from '../blockstore/memory' import { unixfsImporterOptionsDefault } from './constants' import type { PackProperties } from './index' +import {Blockstore} from "../blockstore"; +import {FsBlockStore} from "../blockstore/fs"; export interface PackToStreamProperties extends PackProperties { input: string | Iterable | AsyncIterable, @@ -27,21 +29,32 @@ export async function packToStream ({ input, writable, blockstore: userBlockstor } input = typeof input === 'string' ? [input] : input - const blockstore = userBlockstore ? userBlockstore : new MemoryBlockStore() + if (userBlockstore) { + process.on("SIGINT", async (signal) => await handleSignal(signal, userBlockstore)); + process.on("SIGTERM", async (signal) => await handleSignal(signal, userBlockstore)); + } - // Consume the source - const rootEntry = await last(pipe( - legacyGlobSource(input), - source => normaliseInput(source), - (source: any) => importer(source, blockstore, { - ...unixfsImporterOptionsDefault, - hasher: hasher || unixfsImporterOptionsDefault.hasher, - maxChunkSize: maxChunkSize || unixfsImporterOptionsDefault.maxChunkSize, - maxChildrenPerNode: maxChildrenPerNode || unixfsImporterOptionsDefault.maxChildrenPerNode, - wrapWithDirectory: wrapWithDirectory === false ? false : unixfsImporterOptionsDefault.wrapWithDirectory, - rawLeaves: rawLeaves == null ? unixfsImporterOptionsDefault.rawLeaves : rawLeaves - }) - )) + const blockstore = userBlockstore ? userBlockstore : new MemoryBlockStore() + let rootEntry + + try { + // Consume the source + rootEntry = await last(pipe( + legacyGlobSource(input), + source => normaliseInput(source), + (source: any) => importer(source, blockstore, { + ...unixfsImporterOptionsDefault, + hasher: hasher || unixfsImporterOptionsDefault.hasher, + maxChunkSize: maxChunkSize || unixfsImporterOptionsDefault.maxChunkSize, + maxChildrenPerNode: maxChildrenPerNode || unixfsImporterOptionsDefault.maxChildrenPerNode, + wrapWithDirectory: wrapWithDirectory === false ? false : unixfsImporterOptionsDefault.wrapWithDirectory, + rawLeaves: rawLeaves == null ? unixfsImporterOptionsDefault.rawLeaves : rawLeaves + }) + )) + } catch (err) { + // tslint:disable-next-line:no-console + console.log("Error while importing") + } if (!rootEntry || !rootEntry.cid) { throw new Error('given input could not be parsed correctly') @@ -52,8 +65,13 @@ export async function packToStream ({ input, writable, blockstore: userBlockstor const { writer, out } = await CarWriter.create([root]) Readable.from(out).pipe(writable) - for await (const block of blockstore.blocks()) { - await writer.put(block) + try { + for await (const block of blockstore.blocks()) { + await writer.put(block) + } + } catch (err) { + // tslint:disable-next-line:no-console + console.log("Error while writing blocks") } await writer.close() @@ -86,3 +104,10 @@ async function * legacyGlobSource (input: Iterable | AsyncIterable, roots?: CID[], output?: string, blockstore?: Blockstore}) { - const blockstore = userBlockstore ? userBlockstore : new FsBlockStore() + const blockstore = userBlockstore ? userBlockstore : new FsBlockStore(output ? output : (roots ? roots[0].toString() : "output")) await writeFiles(unpackStream(input, { roots, blockstore }), output) if (!userBlockstore) { await blockstore.close() From 709fd61f8047037bb618ffe060ff5d86961bd727 Mon Sep 17 00:00:00 2001 From: Benjamin Reigner Date: Wed, 4 May 2022 13:17:29 +0200 Subject: [PATCH 2/4] fix: remove the car icon for the process name This car is cute and all but on an operational standpoint it's not practical and even on my own computer with fonts that handle these kinds of characters, the car doesn't display in htop. --- package-lock.json | 1 - package.json | 1 - 2 files changed, 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 2df675a..5d088c9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -32,7 +32,6 @@ "uint8arrays": "^3.0.0" }, "bin": { - "🚘": "dist/cjs/cli/cli.js", "ipfs-car": "dist/cjs/cli/cli.js" }, "devDependencies": { diff --git a/package.json b/package.json index b205221..3161a78 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,6 @@ "/dist" ], "bin": { - "🚘": "./dist/cjs/cli/cli.js", "ipfs-car": "./dist/cjs/cli/cli.js" }, "repository": { From ccbb1793ea1aac62109b1116ba4080faa299bc9b Mon Sep 17 00:00:00 2001 From: Benjamin Reigner Date: Tue, 5 Jul 2022 17:24:51 +0200 Subject: [PATCH 3/4] Revert "fix: remove the car icon for the process name" This reverts commit 709fd61f8047037bb618ffe060ff5d86961bd727. --- package-lock.json | 1 + package.json | 1 + 2 files changed, 2 insertions(+) diff --git a/package-lock.json b/package-lock.json index 5d088c9..2df675a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -32,6 +32,7 @@ "uint8arrays": "^3.0.0" }, "bin": { + "🚘": "dist/cjs/cli/cli.js", "ipfs-car": "dist/cjs/cli/cli.js" }, "devDependencies": { diff --git a/package.json b/package.json index 3161a78..b205221 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,7 @@ "/dist" ], "bin": { + "🚘": "./dist/cjs/cli/cli.js", "ipfs-car": "./dist/cjs/cli/cli.js" }, "repository": { From ad3e2a84f942461ce3c20353d7a199eafc9acd6b Mon Sep 17 00:00:00 2001 From: Benjamin Reigner Date: Tue, 5 Jul 2022 17:29:47 +0200 Subject: [PATCH 4/4] fix: make the path variable in the blockstore class not required This allows to not break custom blockstores and allow at the same time clean kill signal handling. --- src/blockstore/fs.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/blockstore/fs.ts b/src/blockstore/fs.ts index 831e110..43e525a 100644 --- a/src/blockstore/fs.ts +++ b/src/blockstore/fs.ts @@ -3,15 +3,16 @@ import fs from 'fs' import { CID } from 'multiformats' import { BaseBlockstore } from 'blockstore-core' import { Blockstore } from './index' +import * as os from "os"; export class FsBlockStore extends BaseBlockstore implements Blockstore { path: string _opened: boolean _opening?: Promise - constructor (path: string) { + constructor (path?: string) { super() - this.path = path + this.path = path ? path : `${os.tmpdir()}/${(parseInt(String(Math.random() * 1e9), 10)).toString() + Date.now()}` this._opened = false }