mirror of
https://github.com/immich-app/immich.git
synced 2026-05-18 03:10:24 +03:00
refactor: move cli to package folder (#28356)
This commit is contained in:
Jason Rasmussen
@@ -0,0 +1,410 @@
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { setTimeout as sleep } from 'node:timers/promises';
|
||||
import { describe, expect, it, MockedFunction, vi } from 'vitest';
|
||||
|
||||
import { AssetRejectReason, AssetUploadAction, checkBulkUpload, defaults, getSupportedMediaTypes } from '@immich/sdk';
|
||||
import createFetchMock from 'vitest-fetch-mock';
|
||||
|
||||
import {
|
||||
checkForDuplicates,
|
||||
deleteFiles,
|
||||
findSidecar,
|
||||
getAlbumName,
|
||||
startWatch,
|
||||
uploadFiles,
|
||||
UploadOptionsDto,
|
||||
} from 'src/commands/asset';
|
||||
|
||||
vi.mock('@immich/sdk');
|
||||
|
||||
describe('getAlbumName', () => {
|
||||
it('should return a non-undefined value', () => {
|
||||
if (os.platform() === 'win32') {
|
||||
// This is meaningless for Unix systems.
|
||||
expect(getAlbumName(String.raw`D:\test\Filename.txt`, {} as UploadOptionsDto)).toBe('test');
|
||||
}
|
||||
expect(getAlbumName('D:/parentfolder/test/Filename.txt', {} as UploadOptionsDto)).toBe('test');
|
||||
});
|
||||
|
||||
it('has higher priority to return `albumName` in `options`', () => {
|
||||
expect(getAlbumName('/parentfolder/test/Filename.txt', { albumName: 'example' } as UploadOptionsDto)).toBe(
|
||||
'example',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('uploadFiles', () => {
|
||||
const testDir = fs.mkdtempSync(path.join(os.tmpdir(), 'test-'));
|
||||
const testFilePath = path.join(testDir, 'test.png');
|
||||
const testFileData = 'test';
|
||||
const baseUrl = 'http://example.com';
|
||||
const apiKey = 'key';
|
||||
const retry = 3;
|
||||
|
||||
const fetchMocker = createFetchMock(vi);
|
||||
|
||||
beforeEach(() => {
|
||||
// Create a test file
|
||||
fs.writeFileSync(testFilePath, testFileData);
|
||||
|
||||
// Defaults
|
||||
vi.mocked(defaults).baseUrl = baseUrl;
|
||||
vi.mocked(defaults).headers = { 'x-api-key': apiKey };
|
||||
|
||||
fetchMocker.enableMocks();
|
||||
fetchMocker.resetMocks();
|
||||
});
|
||||
|
||||
it('returns new assets when upload file is successful', async () => {
|
||||
fetchMocker.doMockIf(new RegExp(`${baseUrl}/assets$`), function () {
|
||||
return {
|
||||
status: 200,
|
||||
body: JSON.stringify({ id: 'fc5621b1-86f6-44a1-9905-403e607df9f5', status: 'created' }),
|
||||
};
|
||||
});
|
||||
|
||||
await expect(uploadFiles([testFilePath], { concurrency: 1 })).resolves.toEqual([
|
||||
{
|
||||
filepath: testFilePath,
|
||||
id: 'fc5621b1-86f6-44a1-9905-403e607df9f5',
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it('returns new assets when upload file retry is successful', async () => {
|
||||
let counter = 0;
|
||||
fetchMocker.doMockIf(new RegExp(`${baseUrl}/assets$`), function () {
|
||||
counter++;
|
||||
if (counter < retry) {
|
||||
throw new Error('Network error');
|
||||
}
|
||||
|
||||
return {
|
||||
status: 200,
|
||||
body: JSON.stringify({ id: 'fc5621b1-86f6-44a1-9905-403e607df9f5', status: 'created' }),
|
||||
};
|
||||
});
|
||||
|
||||
await expect(uploadFiles([testFilePath], { concurrency: 1 })).resolves.toEqual([
|
||||
{
|
||||
filepath: testFilePath,
|
||||
id: 'fc5621b1-86f6-44a1-9905-403e607df9f5',
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it('returns new assets when upload file retry is failed', async () => {
|
||||
fetchMocker.doMockIf(new RegExp(`${baseUrl}/assets$`), function () {
|
||||
throw new Error('Network error');
|
||||
});
|
||||
|
||||
await expect(uploadFiles([testFilePath], { concurrency: 1 })).resolves.toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('checkForDuplicates', () => {
|
||||
const testDir = fs.mkdtempSync(path.join(os.tmpdir(), 'test-'));
|
||||
const testFilePath = path.join(testDir, 'test.png');
|
||||
const testFileData = 'test';
|
||||
const testFileChecksum = 'a94a8fe5ccb19ba61c4c0873d391e987982fbbd3'; // SHA1
|
||||
const retry = 3;
|
||||
|
||||
beforeEach(() => {
|
||||
// Create a test file
|
||||
fs.writeFileSync(testFilePath, testFileData);
|
||||
});
|
||||
|
||||
it('checks duplicates', async () => {
|
||||
vi.mocked(checkBulkUpload).mockResolvedValue({
|
||||
results: [
|
||||
{
|
||||
action: AssetUploadAction.Accept,
|
||||
id: testFilePath,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
await checkForDuplicates([testFilePath], { concurrency: 1 });
|
||||
|
||||
expect(checkBulkUpload).toHaveBeenCalledWith({
|
||||
assetBulkUploadCheckDto: {
|
||||
assets: [
|
||||
{
|
||||
checksum: testFileChecksum,
|
||||
id: testFilePath,
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('returns duplicates when check duplicates is rejected', async () => {
|
||||
vi.mocked(checkBulkUpload).mockResolvedValue({
|
||||
results: [
|
||||
{
|
||||
action: AssetUploadAction.Reject,
|
||||
id: testFilePath,
|
||||
assetId: 'fc5621b1-86f6-44a1-9905-403e607df9f5',
|
||||
reason: AssetRejectReason.Duplicate,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
await expect(checkForDuplicates([testFilePath], { concurrency: 1 })).resolves.toEqual({
|
||||
duplicates: [
|
||||
{
|
||||
filepath: testFilePath,
|
||||
id: 'fc5621b1-86f6-44a1-9905-403e607df9f5',
|
||||
},
|
||||
],
|
||||
newFiles: [],
|
||||
});
|
||||
});
|
||||
|
||||
it('returns new assets when check duplicates is accepted', async () => {
|
||||
vi.mocked(checkBulkUpload).mockResolvedValue({
|
||||
results: [
|
||||
{
|
||||
action: AssetUploadAction.Accept,
|
||||
id: testFilePath,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
await expect(checkForDuplicates([testFilePath], { concurrency: 1 })).resolves.toEqual({
|
||||
duplicates: [],
|
||||
newFiles: [testFilePath],
|
||||
});
|
||||
});
|
||||
|
||||
it('returns results when check duplicates retry is successful', async () => {
|
||||
let mocked = vi.mocked(checkBulkUpload);
|
||||
for (let i = 1; i < retry; i++) {
|
||||
mocked = mocked.mockRejectedValueOnce(new Error('Network error'));
|
||||
}
|
||||
mocked.mockResolvedValue({
|
||||
results: [
|
||||
{
|
||||
action: AssetUploadAction.Accept,
|
||||
id: testFilePath,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
await expect(checkForDuplicates([testFilePath], { concurrency: 1 })).resolves.toEqual({
|
||||
duplicates: [],
|
||||
newFiles: [testFilePath],
|
||||
});
|
||||
});
|
||||
|
||||
it('returns results when check duplicates retry is failed', async () => {
|
||||
vi.mocked(checkBulkUpload).mockRejectedValue(new Error('Network error'));
|
||||
|
||||
await expect(checkForDuplicates([testFilePath], { concurrency: 1 })).resolves.toEqual({
|
||||
duplicates: [],
|
||||
newFiles: [],
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('startWatch', () => {
|
||||
let testFolder: string;
|
||||
let checkBulkUploadMocked: MockedFunction<typeof checkBulkUpload>;
|
||||
|
||||
beforeEach(async () => {
|
||||
vi.restoreAllMocks();
|
||||
|
||||
vi.mocked(getSupportedMediaTypes).mockResolvedValue({
|
||||
image: ['.jpg'],
|
||||
sidecar: ['.xmp'],
|
||||
video: ['.mp4'],
|
||||
});
|
||||
|
||||
testFolder = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'test-startWatch-'));
|
||||
checkBulkUploadMocked = vi.mocked(checkBulkUpload);
|
||||
checkBulkUploadMocked.mockResolvedValue({
|
||||
results: [],
|
||||
});
|
||||
});
|
||||
|
||||
it('should start watching a directory and upload new files', async () => {
|
||||
const testFilePath = path.join(testFolder, 'test.jpg');
|
||||
|
||||
await startWatch([testFolder], { concurrency: 1 }, { batchSize: 1, debounceTimeMs: 10 });
|
||||
await sleep(100); // to debounce the watcher from considering the test file as a existing file
|
||||
await fs.promises.writeFile(testFilePath, 'testjpg');
|
||||
|
||||
await vi.waitFor(
|
||||
() =>
|
||||
expect(checkBulkUpload).toHaveBeenCalledWith({
|
||||
assetBulkUploadCheckDto: {
|
||||
assets: [
|
||||
expect.objectContaining({
|
||||
id: testFilePath,
|
||||
}),
|
||||
],
|
||||
},
|
||||
}),
|
||||
{ timeout: 5000 },
|
||||
);
|
||||
});
|
||||
|
||||
it('should filter out unsupported files', async () => {
|
||||
const testFilePath = path.join(testFolder, 'test.jpg');
|
||||
const unsupportedFilePath = path.join(testFolder, 'test.txt');
|
||||
|
||||
await startWatch([testFolder], { concurrency: 1 }, { batchSize: 1, debounceTimeMs: 10 });
|
||||
await sleep(100); // to debounce the watcher from considering the test file as a existing file
|
||||
await fs.promises.writeFile(testFilePath, 'testjpg');
|
||||
await fs.promises.writeFile(unsupportedFilePath, 'testtxt');
|
||||
|
||||
await vi.waitFor(
|
||||
() =>
|
||||
expect(checkBulkUpload).toHaveBeenCalledWith({
|
||||
assetBulkUploadCheckDto: {
|
||||
assets: expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
id: testFilePath,
|
||||
}),
|
||||
]),
|
||||
},
|
||||
}),
|
||||
{ timeout: 5000 },
|
||||
);
|
||||
|
||||
expect(checkBulkUpload).not.toHaveBeenCalledWith({
|
||||
assetBulkUploadCheckDto: {
|
||||
assets: expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
id: unsupportedFilePath,
|
||||
}),
|
||||
]),
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('should filter out ignored patterns', async () => {
|
||||
const testFilePath = path.join(testFolder, 'test.jpg');
|
||||
const ignoredPattern = 'ignored';
|
||||
const ignoredFolder = path.join(testFolder, ignoredPattern);
|
||||
await fs.promises.mkdir(ignoredFolder, { recursive: true });
|
||||
const ignoredFilePath = path.join(ignoredFolder, 'ignored.jpg');
|
||||
|
||||
await startWatch([testFolder], { concurrency: 1, ignore: ignoredPattern }, { batchSize: 1, debounceTimeMs: 10 });
|
||||
await sleep(100); // to debounce the watcher from considering the test file as a existing file
|
||||
await fs.promises.writeFile(testFilePath, 'testjpg');
|
||||
await fs.promises.writeFile(ignoredFilePath, 'ignoredjpg');
|
||||
|
||||
await vi.waitFor(
|
||||
() =>
|
||||
expect(checkBulkUpload).toHaveBeenCalledWith({
|
||||
assetBulkUploadCheckDto: {
|
||||
assets: expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
id: testFilePath,
|
||||
}),
|
||||
]),
|
||||
},
|
||||
}),
|
||||
{ timeout: 5000 },
|
||||
);
|
||||
|
||||
expect(checkBulkUpload).not.toHaveBeenCalledWith({
|
||||
assetBulkUploadCheckDto: {
|
||||
assets: expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
id: ignoredFilePath,
|
||||
}),
|
||||
]),
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await fs.promises.rm(testFolder, { recursive: true, force: true });
|
||||
});
|
||||
});
|
||||
|
||||
describe('findSidecar', () => {
|
||||
let testDir: string;
|
||||
let testFilePath: string;
|
||||
|
||||
beforeEach(() => {
|
||||
testDir = fs.mkdtempSync(path.join(os.tmpdir(), 'test-sidecar-'));
|
||||
testFilePath = path.join(testDir, 'test.jpg');
|
||||
fs.writeFileSync(testFilePath, 'test');
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
fs.rmSync(testDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('should find sidecar file with photo.xmp naming convention', () => {
|
||||
const sidecarPath = path.join(testDir, 'test.xmp');
|
||||
fs.writeFileSync(sidecarPath, 'xmp data');
|
||||
|
||||
const result = findSidecar(testFilePath);
|
||||
expect(result).toBe(sidecarPath);
|
||||
});
|
||||
|
||||
it('should find sidecar file with photo.ext.xmp naming convention', () => {
|
||||
const sidecarPath = path.join(testDir, 'test.jpg.xmp');
|
||||
fs.writeFileSync(sidecarPath, 'xmp data');
|
||||
|
||||
const result = findSidecar(testFilePath);
|
||||
expect(result).toBe(sidecarPath);
|
||||
});
|
||||
|
||||
it('should prefer photo.ext.xmp over photo.xmp when both exist', () => {
|
||||
const sidecarPath1 = path.join(testDir, 'test.xmp');
|
||||
const sidecarPath2 = path.join(testDir, 'test.jpg.xmp');
|
||||
fs.writeFileSync(sidecarPath1, 'xmp data 1');
|
||||
fs.writeFileSync(sidecarPath2, 'xmp data 2');
|
||||
|
||||
const result = findSidecar(testFilePath);
|
||||
// Should return the first one found (photo.xmp) based on the order in the code
|
||||
expect(result).toBe(sidecarPath1);
|
||||
});
|
||||
|
||||
it('should return undefined when no sidecar file exists', () => {
|
||||
const result = findSidecar(testFilePath);
|
||||
expect(result).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('deleteFiles', () => {
|
||||
let testDir: string;
|
||||
let testFilePath: string;
|
||||
|
||||
beforeEach(() => {
|
||||
testDir = fs.mkdtempSync(path.join(os.tmpdir(), 'test-delete-'));
|
||||
testFilePath = path.join(testDir, 'test.jpg');
|
||||
fs.writeFileSync(testFilePath, 'test');
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
fs.rmSync(testDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('should delete asset and sidecar file when main file is deleted', async () => {
|
||||
const sidecarPath = path.join(testDir, 'test.xmp');
|
||||
fs.writeFileSync(sidecarPath, 'xmp data');
|
||||
|
||||
await deleteFiles([{ id: 'test-id', filepath: testFilePath }], [], { delete: true, concurrency: 1 });
|
||||
|
||||
expect(fs.existsSync(testFilePath)).toBe(false);
|
||||
expect(fs.existsSync(sidecarPath)).toBe(false);
|
||||
});
|
||||
|
||||
it('should not delete sidecar file when delete option is false', async () => {
|
||||
const sidecarPath = path.join(testDir, 'test.xmp');
|
||||
fs.writeFileSync(sidecarPath, 'xmp data');
|
||||
|
||||
await deleteFiles([{ id: 'test-id', filepath: testFilePath }], [], { delete: false, concurrency: 1 });
|
||||
|
||||
expect(fs.existsSync(testFilePath)).toBe(true);
|
||||
expect(fs.existsSync(sidecarPath)).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,587 @@
|
||||
import {
|
||||
AssetBulkUploadCheckItem,
|
||||
AssetBulkUploadCheckResult,
|
||||
AssetMediaResponseDto,
|
||||
AssetMediaStatus,
|
||||
AssetUploadAction,
|
||||
Permission,
|
||||
addAssetsToAlbum,
|
||||
checkBulkUpload,
|
||||
createAlbum,
|
||||
defaults,
|
||||
getAllAlbums,
|
||||
getSupportedMediaTypes,
|
||||
} from '@immich/sdk';
|
||||
import byteSize from 'byte-size';
|
||||
import { Matcher, watch as watchFs } from 'chokidar';
|
||||
import { MultiBar, Presets, SingleBar } from 'cli-progress';
|
||||
import { chunk } from 'lodash-es';
|
||||
import micromatch from 'micromatch';
|
||||
import { Stats, createReadStream, existsSync } from 'node:fs';
|
||||
import { stat, unlink } from 'node:fs/promises';
|
||||
import path, { basename } from 'node:path';
|
||||
import { Queue } from 'src/queue';
|
||||
import { BaseOptions, Batcher, authenticate, crawl, requirePermissions, s, sha1 } from 'src/utils';
|
||||
|
||||
const UPLOAD_WATCH_BATCH_SIZE = 100;
|
||||
const UPLOAD_WATCH_DEBOUNCE_TIME_MS = 10_000;
|
||||
|
||||
// TODO figure out why `id` is missing
|
||||
type AssetBulkUploadCheckResults = Array<AssetBulkUploadCheckResult & { id: string }>;
|
||||
type Asset = { id: string; filepath: string };
|
||||
|
||||
export interface UploadOptionsDto {
|
||||
recursive?: boolean;
|
||||
ignore?: string;
|
||||
dryRun?: boolean;
|
||||
skipHash?: boolean;
|
||||
delete?: boolean;
|
||||
deleteDuplicates?: boolean;
|
||||
album?: boolean;
|
||||
albumName?: string;
|
||||
includeHidden?: boolean;
|
||||
concurrency: number;
|
||||
progress?: boolean;
|
||||
watch?: boolean;
|
||||
jsonOutput?: boolean;
|
||||
}
|
||||
|
||||
class UploadFile extends File {
|
||||
constructor(
|
||||
private filepath: string,
|
||||
private _size: number,
|
||||
) {
|
||||
super([], basename(filepath));
|
||||
}
|
||||
|
||||
get size() {
|
||||
return this._size;
|
||||
}
|
||||
|
||||
stream() {
|
||||
return createReadStream(this.filepath) as any;
|
||||
}
|
||||
}
|
||||
|
||||
const uploadBatch = async (files: string[], options: UploadOptionsDto) => {
|
||||
const { newFiles, duplicates } = await checkForDuplicates(files, options);
|
||||
const newAssets = await uploadFiles(newFiles, options);
|
||||
if (options.jsonOutput) {
|
||||
console.log(JSON.stringify({ newFiles, duplicates, newAssets }, undefined, 4));
|
||||
}
|
||||
await updateAlbums([...newAssets, ...duplicates], options);
|
||||
|
||||
await deleteFiles(newAssets, duplicates, options);
|
||||
};
|
||||
|
||||
export const startWatch = async (
|
||||
paths: string[],
|
||||
options: UploadOptionsDto,
|
||||
{
|
||||
batchSize = UPLOAD_WATCH_BATCH_SIZE,
|
||||
debounceTimeMs = UPLOAD_WATCH_DEBOUNCE_TIME_MS,
|
||||
}: { batchSize?: number; debounceTimeMs?: number } = {},
|
||||
) => {
|
||||
const watcherIgnored: Matcher[] = [];
|
||||
const { image, video } = await getSupportedMediaTypes();
|
||||
const extensions = new Set([...image, ...video]);
|
||||
|
||||
if (options.ignore) {
|
||||
watcherIgnored.push((path) => micromatch.contains(path, `**/${options.ignore}`));
|
||||
}
|
||||
|
||||
const pathsBatcher = new Batcher<string>({
|
||||
batchSize,
|
||||
debounceTimeMs,
|
||||
onBatch: async (paths: string[]) => {
|
||||
const uniquePaths = [...new Set(paths)];
|
||||
await uploadBatch(uniquePaths, options);
|
||||
},
|
||||
});
|
||||
|
||||
const onFile = async (path: string, stats?: Stats) => {
|
||||
if (stats?.isDirectory()) {
|
||||
return;
|
||||
}
|
||||
const ext = '.' + path.split('.').pop()?.toLowerCase();
|
||||
if (!ext || !extensions.has(ext)) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!options.progress) {
|
||||
// logging when progress is disabled as it can cause issues with the progress bar rendering
|
||||
console.log(`Change detected: ${path}`);
|
||||
}
|
||||
pathsBatcher.add(path);
|
||||
};
|
||||
const fsWatcher = watchFs(paths, {
|
||||
ignoreInitial: true,
|
||||
ignored: watcherIgnored,
|
||||
alwaysStat: true,
|
||||
awaitWriteFinish: true,
|
||||
depth: options.recursive ? undefined : 1,
|
||||
persistent: true,
|
||||
})
|
||||
.on('add', onFile)
|
||||
.on('change', onFile)
|
||||
.on('error', (error) => console.error(`Watcher error: ${error}`));
|
||||
|
||||
process.on('SIGINT', async () => {
|
||||
console.log('Exiting...');
|
||||
await fsWatcher.close();
|
||||
process.exit();
|
||||
});
|
||||
};
|
||||
|
||||
export const upload = async (paths: string[], baseOptions: BaseOptions, options: UploadOptionsDto) => {
|
||||
await authenticate(baseOptions);
|
||||
await requirePermissions([Permission.AssetUpload]);
|
||||
|
||||
const scanFiles = await scan(paths, options);
|
||||
|
||||
if (scanFiles.length === 0) {
|
||||
if (options.watch) {
|
||||
console.log('No files found initially.');
|
||||
} else {
|
||||
console.log('No files found, exiting');
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (options.watch) {
|
||||
console.log('Watching for changes...');
|
||||
await startWatch(paths, options);
|
||||
// watcher does not handle the initial scan
|
||||
// as the scan() is a more efficient quick start with batched results
|
||||
}
|
||||
|
||||
await uploadBatch(scanFiles, options);
|
||||
};
|
||||
|
||||
const scan = async (pathsToCrawl: string[], options: UploadOptionsDto) => {
|
||||
const { image, video } = await getSupportedMediaTypes();
|
||||
|
||||
console.log('Crawling for assets...');
|
||||
const files = await crawl({
|
||||
pathsToCrawl,
|
||||
recursive: options.recursive,
|
||||
exclusionPattern: options.ignore,
|
||||
includeHidden: options.includeHidden,
|
||||
extensions: [...image, ...video],
|
||||
});
|
||||
|
||||
return files;
|
||||
};
|
||||
|
||||
export const checkForDuplicates = async (files: string[], { concurrency, skipHash, progress }: UploadOptionsDto) => {
|
||||
if (skipHash) {
|
||||
console.log('Skipping hash check, assuming all files are new');
|
||||
return { newFiles: files, duplicates: [] };
|
||||
}
|
||||
|
||||
let multiBar: MultiBar | undefined;
|
||||
let totalSize = 0;
|
||||
const statsMap = new Map<string, Stats>();
|
||||
|
||||
// Calculate total size first
|
||||
for (const filepath of files) {
|
||||
const stats = await stat(filepath);
|
||||
statsMap.set(filepath, stats);
|
||||
totalSize += stats.size;
|
||||
}
|
||||
|
||||
if (progress) {
|
||||
multiBar = new MultiBar(
|
||||
{
|
||||
format: '{message} | {bar} | {percentage}% | ETA: {eta_formatted} | {value}/{total}',
|
||||
formatValue: (v: number, options, type) => {
|
||||
// Don't format percentage
|
||||
if (type === 'percentage') {
|
||||
return v.toString();
|
||||
}
|
||||
return byteSize(v).toString();
|
||||
},
|
||||
etaBuffer: 100, // Increase samples for ETA calculation
|
||||
},
|
||||
Presets.shades_classic,
|
||||
);
|
||||
|
||||
// Ensure we restore cursor on interrupt
|
||||
process.on('SIGINT', () => {
|
||||
if (multiBar) {
|
||||
multiBar.stop();
|
||||
}
|
||||
process.exit(0);
|
||||
});
|
||||
} else {
|
||||
console.log(`Received ${files.length} files (${byteSize(totalSize)}), hashing...`);
|
||||
}
|
||||
|
||||
const hashProgressBar = multiBar?.create(totalSize, 0, {
|
||||
message: 'Hashing files ',
|
||||
});
|
||||
const checkProgressBar = multiBar?.create(totalSize, 0, {
|
||||
message: 'Checking for duplicates',
|
||||
});
|
||||
|
||||
const newFiles: string[] = [];
|
||||
const duplicates: Asset[] = [];
|
||||
|
||||
const checkBulkUploadQueue = new Queue<AssetBulkUploadCheckItem[], void>(
|
||||
async (assets: AssetBulkUploadCheckItem[]) => {
|
||||
const response = await checkBulkUpload({ assetBulkUploadCheckDto: { assets } });
|
||||
|
||||
const results = response.results as AssetBulkUploadCheckResults;
|
||||
|
||||
for (const { id: filepath, assetId, action } of results) {
|
||||
if (action === AssetUploadAction.Accept) {
|
||||
newFiles.push(filepath);
|
||||
} else {
|
||||
// rejects are always duplicates
|
||||
duplicates.push({ id: assetId as string, filepath });
|
||||
}
|
||||
}
|
||||
|
||||
// Update progress based on total size of processed files
|
||||
let processedSize = 0;
|
||||
for (const asset of assets) {
|
||||
const stats = statsMap.get(asset.id);
|
||||
processedSize += stats?.size || 0;
|
||||
}
|
||||
checkProgressBar?.increment(processedSize);
|
||||
},
|
||||
{ concurrency, retry: 3 },
|
||||
);
|
||||
|
||||
const results: { id: string; checksum: string }[] = [];
|
||||
let checkBulkUploadRequests: AssetBulkUploadCheckItem[] = [];
|
||||
|
||||
const queue = new Queue<string, AssetBulkUploadCheckItem[]>(
|
||||
async (filepath: string): Promise<AssetBulkUploadCheckItem[]> => {
|
||||
const stats = statsMap.get(filepath);
|
||||
if (!stats) {
|
||||
throw new Error(`Stats not found for ${filepath}`);
|
||||
}
|
||||
const dto = { id: filepath, checksum: await sha1(filepath) };
|
||||
|
||||
results.push(dto);
|
||||
checkBulkUploadRequests.push(dto);
|
||||
if (checkBulkUploadRequests.length === 5000) {
|
||||
const batch = checkBulkUploadRequests;
|
||||
checkBulkUploadRequests = [];
|
||||
void checkBulkUploadQueue.push(batch);
|
||||
}
|
||||
|
||||
hashProgressBar?.increment(stats.size);
|
||||
return results;
|
||||
},
|
||||
{ concurrency, retry: 3 },
|
||||
);
|
||||
|
||||
for (const item of files) {
|
||||
void queue.push(item);
|
||||
}
|
||||
|
||||
await queue.drained();
|
||||
|
||||
if (checkBulkUploadRequests.length > 0) {
|
||||
void checkBulkUploadQueue.push(checkBulkUploadRequests);
|
||||
}
|
||||
|
||||
await checkBulkUploadQueue.drained();
|
||||
|
||||
multiBar?.stop();
|
||||
|
||||
console.log(`Found ${newFiles.length} new files and ${duplicates.length} duplicate${s(duplicates.length)}`);
|
||||
|
||||
// Report failures
|
||||
const failedTasks = queue.tasks.filter((task) => task.status === 'failed');
|
||||
if (failedTasks.length > 0) {
|
||||
console.log(`Failed to verify ${failedTasks.length} file${s(failedTasks.length)}:`);
|
||||
for (const task of failedTasks) {
|
||||
console.log(`- ${task.data} - ${task.error}`);
|
||||
}
|
||||
}
|
||||
|
||||
return { newFiles, duplicates };
|
||||
};
|
||||
|
||||
export const uploadFiles = async (
|
||||
files: string[],
|
||||
{ dryRun, concurrency, progress }: UploadOptionsDto,
|
||||
): Promise<Asset[]> => {
|
||||
if (files.length === 0) {
|
||||
console.log('All assets were already uploaded, nothing to do.');
|
||||
return [];
|
||||
}
|
||||
|
||||
// Compute total size first
|
||||
let totalSize = 0;
|
||||
const statsMap = new Map<string, Stats>();
|
||||
for (const filepath of files) {
|
||||
const stats = await stat(filepath);
|
||||
statsMap.set(filepath, stats);
|
||||
totalSize += stats.size;
|
||||
}
|
||||
|
||||
if (dryRun) {
|
||||
console.log(`Would have uploaded ${files.length} asset${s(files.length)} (${byteSize(totalSize)})`);
|
||||
return files.map((filepath) => ({ id: '', filepath }));
|
||||
}
|
||||
|
||||
let uploadProgress: SingleBar | undefined;
|
||||
|
||||
if (progress) {
|
||||
uploadProgress = new SingleBar(
|
||||
{
|
||||
format: 'Uploading assets | {bar} | {percentage}% | ETA: {eta_formatted} | {value_formatted}/{total_formatted}',
|
||||
},
|
||||
Presets.shades_classic,
|
||||
);
|
||||
} else {
|
||||
console.log(`Uploading ${files.length} asset${s(files.length)} (${byteSize(totalSize)})`);
|
||||
}
|
||||
uploadProgress?.start(totalSize, 0);
|
||||
uploadProgress?.update({ value_formatted: 0, total_formatted: byteSize(totalSize) });
|
||||
|
||||
let duplicateCount = 0;
|
||||
let duplicateSize = 0;
|
||||
let successCount = 0;
|
||||
let successSize = 0;
|
||||
|
||||
const newAssets: Asset[] = [];
|
||||
|
||||
const queue = new Queue<string, AssetMediaResponseDto>(
|
||||
async (filepath: string) => {
|
||||
const stats = statsMap.get(filepath);
|
||||
if (!stats) {
|
||||
throw new Error(`Stats not found for ${filepath}`);
|
||||
}
|
||||
|
||||
const response = await uploadFile(filepath, stats);
|
||||
newAssets.push({ id: response.id, filepath });
|
||||
if (response.status === AssetMediaStatus.Duplicate) {
|
||||
duplicateCount++;
|
||||
duplicateSize += stats.size ?? 0;
|
||||
} else {
|
||||
successCount++;
|
||||
successSize += stats.size ?? 0;
|
||||
}
|
||||
|
||||
uploadProgress?.update(successSize, { value_formatted: byteSize(successSize + duplicateSize) });
|
||||
|
||||
return response;
|
||||
},
|
||||
{ concurrency, retry: 3 },
|
||||
);
|
||||
|
||||
for (const item of files) {
|
||||
void queue.push(item);
|
||||
}
|
||||
|
||||
await queue.drained();
|
||||
|
||||
uploadProgress?.stop();
|
||||
|
||||
console.log(`Successfully uploaded ${successCount} new asset${s(successCount)} (${byteSize(successSize)})`);
|
||||
if (duplicateCount > 0) {
|
||||
console.log(`Skipped ${duplicateCount} duplicate asset${s(duplicateCount)} (${byteSize(duplicateSize)})`);
|
||||
}
|
||||
|
||||
// Report failures
|
||||
const failedTasks = queue.tasks.filter((task) => task.status === 'failed');
|
||||
if (failedTasks.length > 0) {
|
||||
console.log(`Failed to upload ${failedTasks.length} asset${s(failedTasks.length)}:`);
|
||||
for (const task of failedTasks) {
|
||||
console.log(`- ${task.data} - ${task.error}`);
|
||||
}
|
||||
}
|
||||
|
||||
return newAssets;
|
||||
};
|
||||
|
||||
const uploadFile = async (input: string, stats: Stats): Promise<AssetMediaResponseDto> => {
|
||||
const { baseUrl, headers } = defaults;
|
||||
|
||||
const formData = new FormData();
|
||||
formData.append('fileCreatedAt', stats.mtime.toISOString());
|
||||
formData.append('fileModifiedAt', stats.mtime.toISOString());
|
||||
formData.append('fileSize', String(stats.size));
|
||||
formData.append('isFavorite', 'false');
|
||||
formData.append('assetData', new UploadFile(input, stats.size));
|
||||
|
||||
const sidecarPath = findSidecar(input);
|
||||
if (sidecarPath) {
|
||||
try {
|
||||
const stats = await stat(sidecarPath);
|
||||
const sidecarData = new UploadFile(sidecarPath, stats.size);
|
||||
formData.append('sidecarData', sidecarData);
|
||||
} catch {
|
||||
// noop
|
||||
}
|
||||
}
|
||||
|
||||
const response = await fetch(`${baseUrl}/assets`, {
|
||||
method: 'post',
|
||||
redirect: 'error',
|
||||
headers: headers as Record<string, string>,
|
||||
body: formData,
|
||||
});
|
||||
if (response.status !== 200 && response.status !== 201) {
|
||||
throw new Error(await response.text());
|
||||
}
|
||||
|
||||
return response.json();
|
||||
};
|
||||
|
||||
export const findSidecar = (filepath: string): string | undefined => {
|
||||
const assetPath = path.parse(filepath);
|
||||
const noExtension = path.join(assetPath.dir, assetPath.name);
|
||||
|
||||
// XMP sidecars can come in two filename formats. For a photo named photo.ext, the filenames are photo.ext.xmp and photo.xmp
|
||||
for (const sidecarPath of [`${noExtension}.xmp`, `${filepath}.xmp`]) {
|
||||
if (existsSync(sidecarPath)) {
|
||||
return sidecarPath;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
export const deleteFiles = async (uploaded: Asset[], duplicates: Asset[], options: UploadOptionsDto): Promise<void> => {
|
||||
let fileCount = 0;
|
||||
if (options.delete) {
|
||||
fileCount += uploaded.length;
|
||||
}
|
||||
|
||||
if (options.deleteDuplicates) {
|
||||
fileCount += duplicates.length;
|
||||
}
|
||||
|
||||
if (options.dryRun) {
|
||||
console.log(`Would have deleted ${fileCount} local asset${s(fileCount)}`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (fileCount === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
console.log('Deleting assets that have been uploaded...');
|
||||
const deletionProgress = new SingleBar(
|
||||
{ format: 'Deleting local assets | {bar} | {percentage}% | ETA: {eta}s | {value}/{total} assets' },
|
||||
Presets.shades_classic,
|
||||
);
|
||||
deletionProgress.start(fileCount, 0);
|
||||
|
||||
const chunkDelete = async (files: Asset[]) => {
|
||||
for (const assetBatch of chunk(files, options.concurrency)) {
|
||||
await Promise.all(
|
||||
assetBatch.map(async (input: Asset) => {
|
||||
await unlink(input.filepath);
|
||||
const sidecarPath = findSidecar(input.filepath);
|
||||
if (sidecarPath) {
|
||||
await unlink(sidecarPath);
|
||||
}
|
||||
}),
|
||||
);
|
||||
deletionProgress.update(assetBatch.length);
|
||||
}
|
||||
};
|
||||
|
||||
try {
|
||||
if (options.delete) {
|
||||
await chunkDelete(uploaded);
|
||||
}
|
||||
|
||||
if (options.deleteDuplicates) {
|
||||
await chunkDelete(duplicates);
|
||||
}
|
||||
} finally {
|
||||
deletionProgress.stop();
|
||||
}
|
||||
};
|
||||
|
||||
const updateAlbums = async (assets: Asset[], options: UploadOptionsDto) => {
|
||||
if (!options.album && !options.albumName) {
|
||||
return;
|
||||
}
|
||||
const { dryRun, concurrency } = options;
|
||||
|
||||
const albums = await getAllAlbums({});
|
||||
const existingAlbums = new Map(albums.map((album) => [album.albumName, album.id]));
|
||||
const newAlbums: Set<string> = new Set();
|
||||
for (const { filepath } of assets) {
|
||||
const albumName = getAlbumName(filepath, options);
|
||||
if (albumName && !existingAlbums.has(albumName)) {
|
||||
newAlbums.add(albumName);
|
||||
}
|
||||
}
|
||||
|
||||
if (dryRun) {
|
||||
// TODO print asset counts for new albums
|
||||
console.log(`Would have created ${newAlbums.size} new album${s(newAlbums.size)}`);
|
||||
console.log(`Would have updated albums of ${assets.length} asset${s(assets.length)}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const progressBar = new SingleBar(
|
||||
{ format: 'Creating albums | {bar} | {percentage}% | ETA: {eta}s | {value}/{total} albums' },
|
||||
Presets.shades_classic,
|
||||
);
|
||||
progressBar.start(newAlbums.size, 0);
|
||||
|
||||
try {
|
||||
for (const albumNames of chunk([...newAlbums], concurrency)) {
|
||||
const items = await Promise.all(
|
||||
albumNames.map((albumName: string) => createAlbum({ createAlbumDto: { albumName } })),
|
||||
);
|
||||
for (const { id, albumName } of items) {
|
||||
existingAlbums.set(albumName, id);
|
||||
}
|
||||
progressBar.increment(albumNames.length);
|
||||
}
|
||||
} finally {
|
||||
progressBar.stop();
|
||||
}
|
||||
|
||||
console.log(`Successfully created ${newAlbums.size} new album${s(newAlbums.size)}`);
|
||||
console.log(`Successfully updated ${assets.length} asset${s(assets.length)}`);
|
||||
|
||||
const albumToAssets = new Map<string, string[]>();
|
||||
for (const asset of assets) {
|
||||
const albumName = getAlbumName(asset.filepath, options);
|
||||
if (!albumName) {
|
||||
continue;
|
||||
}
|
||||
const albumId = existingAlbums.get(albumName);
|
||||
if (albumId) {
|
||||
if (!albumToAssets.has(albumId)) {
|
||||
albumToAssets.set(albumId, []);
|
||||
}
|
||||
albumToAssets.get(albumId)?.push(asset.id);
|
||||
}
|
||||
}
|
||||
|
||||
const albumUpdateProgress = new SingleBar(
|
||||
{ format: 'Adding assets to albums | {bar} | {percentage}% | ETA: {eta}s | {value}/{total} assets' },
|
||||
Presets.shades_classic,
|
||||
);
|
||||
albumUpdateProgress.start(assets.length, 0);
|
||||
|
||||
try {
|
||||
for (const [albumId, assets] of albumToAssets.entries()) {
|
||||
for (const assetBatch of chunk(assets, Math.min(1000 * concurrency, 65_000))) {
|
||||
await addAssetsToAlbum({ id: albumId, bulkIdsDto: { ids: assetBatch } });
|
||||
albumUpdateProgress.increment(assetBatch.length);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
albumUpdateProgress.stop();
|
||||
}
|
||||
};
|
||||
|
||||
// `filepath` valid format:
|
||||
// - Windows: `D:\\test\\Filename.txt` or `D:/test/Filename.txt`
|
||||
// - Unix: `/test/Filename.txt`
|
||||
export const getAlbumName = (filepath: string, options: UploadOptionsDto) => {
|
||||
return options.albumName ?? path.basename(path.dirname(filepath));
|
||||
};
|
||||
@@ -0,0 +1,57 @@
|
||||
import { getMyUser, Permission } from '@immich/sdk';
|
||||
import { existsSync } from 'node:fs';
|
||||
import { mkdir, unlink } from 'node:fs/promises';
|
||||
import {
|
||||
BaseOptions,
|
||||
connect,
|
||||
getAuthFilePath,
|
||||
logError,
|
||||
requirePermissions,
|
||||
withError,
|
||||
writeAuthFile,
|
||||
} from 'src/utils';
|
||||
|
||||
export const login = async (url: string, key: string, options: BaseOptions) => {
|
||||
console.log(`Logging in to ${url}`);
|
||||
|
||||
const { configDirectory: configDir } = options;
|
||||
|
||||
await connect(url, key);
|
||||
await requirePermissions([Permission.UserRead]);
|
||||
|
||||
const [error, user] = await withError(getMyUser());
|
||||
if (error) {
|
||||
logError(error, 'Failed to load user info');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log(`Logged in as ${user.email}`);
|
||||
|
||||
if (!existsSync(configDir)) {
|
||||
// Create config folder if it doesn't exist
|
||||
const created = await mkdir(configDir, { recursive: true });
|
||||
if (!created) {
|
||||
console.log(`Failed to create config folder: ${configDir}`);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
await writeAuthFile(configDir, { url, key });
|
||||
|
||||
console.log(`Wrote auth info to ${getAuthFilePath(configDir)}`);
|
||||
};
|
||||
|
||||
export const logout = async (options: BaseOptions) => {
|
||||
console.log('Logging out...');
|
||||
|
||||
const { configDirectory: configDir } = options;
|
||||
|
||||
const authFile = getAuthFilePath(configDir);
|
||||
|
||||
if (existsSync(authFile)) {
|
||||
await unlink(authFile);
|
||||
console.log(`Removed auth file: ${authFile}`);
|
||||
}
|
||||
|
||||
console.log('Successfully logged out');
|
||||
};
|
||||
@@ -0,0 +1,25 @@
|
||||
import { getAssetStatistics, getMyUser, getServerVersion, getSupportedMediaTypes, Permission } from '@immich/sdk';
|
||||
import { authenticate, BaseOptions, requirePermissions } from 'src/utils';
|
||||
|
||||
export const serverInfo = async (options: BaseOptions) => {
|
||||
const { url } = await authenticate(options);
|
||||
await requirePermissions([Permission.ServerAbout, Permission.AssetStatistics, Permission.UserRead]);
|
||||
|
||||
const [versionInfo, mediaTypes, stats, userInfo] = await Promise.all([
|
||||
getServerVersion(),
|
||||
getSupportedMediaTypes(),
|
||||
getAssetStatistics({}),
|
||||
getMyUser(),
|
||||
]);
|
||||
|
||||
console.log(`Server Info (via ${userInfo.email})`);
|
||||
console.log(` Url: ${url}`);
|
||||
console.log(` Version: ${versionInfo.major}.${versionInfo.minor}.${versionInfo.patch}`);
|
||||
console.log(` Formats:`);
|
||||
console.log(` Images: ${mediaTypes.image.map((extension) => extension.replace('.', ''))}`);
|
||||
console.log(` Videos: ${mediaTypes.video.map((extension) => extension.replace('.', ''))}`);
|
||||
console.log(` Statistics:`);
|
||||
console.log(` Images: ${stats.images}`);
|
||||
console.log(` Videos: ${stats.videos}`);
|
||||
console.log(` Total: ${stats.total}`);
|
||||
};
|
||||
@@ -0,0 +1,93 @@
|
||||
#! /usr/bin/env node
|
||||
import { Command, Option } from 'commander';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { upload } from 'src/commands/asset';
|
||||
import { login, logout } from 'src/commands/auth';
|
||||
import { serverInfo } from 'src/commands/server-info';
|
||||
import { version } from '../package.json';
|
||||
|
||||
const defaultConfigDirectory = path.join(os.homedir(), '.config/immich/');
|
||||
const defaultConcurrency = Math.max(1, os.cpus().length - 1);
|
||||
|
||||
const program = new Command()
|
||||
.name('immich')
|
||||
.version(version)
|
||||
.description('Command line interface for Immich')
|
||||
.addOption(
|
||||
new Option('-d, --config-directory <directory>', 'Configuration directory where auth.yml will be stored')
|
||||
.env('IMMICH_CONFIG_DIR')
|
||||
.default(defaultConfigDirectory),
|
||||
)
|
||||
.addOption(new Option('-u, --url [url]', 'Immich server URL').env('IMMICH_INSTANCE_URL'))
|
||||
.addOption(new Option('-k, --key [key]', 'Immich API key').env('IMMICH_API_KEY'));
|
||||
|
||||
program
|
||||
.command('login')
|
||||
.alias('login-key')
|
||||
.description('Login using an API key')
|
||||
.argument('url', 'Immich server URL')
|
||||
.argument('key', 'Immich API key')
|
||||
.action((url, key) => login(url, key, program.opts()));
|
||||
|
||||
program
|
||||
.command('logout')
|
||||
.description('Remove stored credentials')
|
||||
.action(() => logout(program.opts()));
|
||||
|
||||
program
|
||||
.command('server-info')
|
||||
.description('Display server information')
|
||||
.action(() => serverInfo(program.opts()));
|
||||
|
||||
program
|
||||
.command('upload')
|
||||
.description('Upload assets')
|
||||
.usage('[paths...] [options]')
|
||||
.addOption(new Option('-r, --recursive', 'Recursive').env('IMMICH_RECURSIVE').default(false))
|
||||
.addOption(new Option('-i, --ignore <pattern>', 'Pattern to ignore').env('IMMICH_IGNORE_PATHS'))
|
||||
.addOption(new Option('-h, --skip-hash', "Don't hash files before upload").env('IMMICH_SKIP_HASH').default(false))
|
||||
.addOption(new Option('-H, --include-hidden', 'Include hidden folders').env('IMMICH_INCLUDE_HIDDEN').default(false))
|
||||
.addOption(
|
||||
new Option('-a, --album', 'Automatically create albums based on folder name')
|
||||
.env('IMMICH_AUTO_CREATE_ALBUM')
|
||||
.default(false),
|
||||
)
|
||||
.addOption(
|
||||
new Option('-A, --album-name <name>', 'Add all assets to specified album')
|
||||
.env('IMMICH_ALBUM_NAME')
|
||||
.conflicts('album'),
|
||||
)
|
||||
.addOption(
|
||||
new Option('-n, --dry-run', "Don't perform any actions, just show what will be done")
|
||||
.env('IMMICH_DRY_RUN')
|
||||
.default(false)
|
||||
.conflicts('skipHash'),
|
||||
)
|
||||
.addOption(
|
||||
new Option('-c, --concurrency <number>', 'Number of assets to upload at the same time')
|
||||
.env('IMMICH_UPLOAD_CONCURRENCY')
|
||||
.default(defaultConcurrency),
|
||||
)
|
||||
.addOption(
|
||||
new Option('-j, --json-output', 'Output detailed information in json format')
|
||||
.env('IMMICH_JSON_OUTPUT')
|
||||
.default(false),
|
||||
)
|
||||
.addOption(new Option('--delete', 'Delete local assets after upload').env('IMMICH_DELETE_ASSETS'))
|
||||
.addOption(
|
||||
new Option('--delete-duplicates', 'Delete local assets that are duplicates (already exist on server)').env(
|
||||
'IMMICH_DELETE_DUPLICATES',
|
||||
),
|
||||
)
|
||||
.addOption(new Option('--no-progress', 'Hide progress bars').env('IMMICH_PROGRESS_BAR').default(true))
|
||||
.addOption(
|
||||
new Option('--watch', 'Watch for changes and upload automatically')
|
||||
.env('IMMICH_WATCH_CHANGES')
|
||||
.default(false)
|
||||
.implies({ progress: false }),
|
||||
)
|
||||
.argument('[paths...]', 'One or more paths to assets to be uploaded')
|
||||
.action((paths, options) => upload(paths, program.opts(), options));
|
||||
|
||||
program.parse(process.argv);
|
||||
@@ -0,0 +1,131 @@
|
||||
import * as fastq from 'fastq';
|
||||
import { uniqueId } from 'lodash-es';
|
||||
|
||||
export type Task<T, R> = {
|
||||
readonly id: string;
|
||||
status: 'idle' | 'processing' | 'succeeded' | 'failed';
|
||||
data: T;
|
||||
error: unknown | undefined;
|
||||
count: number;
|
||||
// TODO: Could be useful to adding progress property.
|
||||
// TODO: Could be useful to adding start_at/end_at/duration properties.
|
||||
result: undefined | R;
|
||||
};
|
||||
|
||||
export type QueueOptions = {
|
||||
verbose?: boolean;
|
||||
concurrency?: number;
|
||||
retry?: number;
|
||||
// TODO: Could be useful to adding timeout property for retry.
|
||||
};
|
||||
|
||||
export type ComputedQueueOptions = Required<QueueOptions>;
|
||||
|
||||
export const defaultQueueOptions = {
|
||||
concurrency: 1,
|
||||
retry: 0,
|
||||
verbose: false,
|
||||
};
|
||||
|
||||
/**
|
||||
* An in-memory queue that processes tasks in parallel with a given concurrency.
|
||||
* @see {@link https://www.npmjs.com/package/fastq}
|
||||
* @template T - The type of the worker task data.
|
||||
* @template R - The type of the worker output data.
|
||||
*/
|
||||
export class Queue<T, R> {
|
||||
private readonly queue: fastq.queueAsPromised<string, Task<T, R>>;
|
||||
private readonly store = new Map<string, Task<T, R>>();
|
||||
readonly options: ComputedQueueOptions;
|
||||
readonly worker: (data: T) => Promise<R>;
|
||||
|
||||
/**
|
||||
* Create a new queue.
|
||||
* @param worker - The worker function that processes the task.
|
||||
* @param options - The queue options.
|
||||
*/
|
||||
constructor(worker: (data: T) => Promise<R>, options?: QueueOptions) {
|
||||
this.options = { ...defaultQueueOptions, ...options };
|
||||
this.worker = worker;
|
||||
this.store = new Map<string, Task<T, R>>();
|
||||
this.queue = this.buildQueue();
|
||||
}
|
||||
|
||||
get tasks(): Task<T, R>[] {
|
||||
const tasks: Task<T, R>[] = [];
|
||||
for (const task of this.store.values()) {
|
||||
tasks.push(task);
|
||||
}
|
||||
return tasks;
|
||||
}
|
||||
|
||||
getTask(id: string): Task<T, R> {
|
||||
const task = this.store.get(id);
|
||||
if (!task) {
|
||||
throw new Error(`Task with id ${id} not found`);
|
||||
}
|
||||
return task;
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait for the queue to be empty.
|
||||
* @returns Promise<void> - The returned Promise will be resolved when all tasks in the queue have been processed by a worker.
|
||||
* This promise could be ignored as it will not lead to a `unhandledRejection`.
|
||||
*/
|
||||
drained(): Promise<void> {
|
||||
return this.queue.drained();
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a task at the end of the queue.
|
||||
* @see {@link https://www.npmjs.com/package/fastq}
|
||||
* @param data
|
||||
* @returns Promise<void> - A Promise that will be fulfilled (rejected) when the task is completed successfully (unsuccessfully).
|
||||
* This promise could be ignored as it will not lead to a `unhandledRejection`.
|
||||
*/
|
||||
async push(data: T): Promise<Task<T, R>> {
|
||||
const id = uniqueId();
|
||||
const task: Task<T, R> = { id, status: 'idle', error: undefined, count: 0, data, result: undefined };
|
||||
this.store.set(id, task);
|
||||
return this.queue.push(id);
|
||||
}
|
||||
|
||||
// TODO: Support more function delegation to fastq.
|
||||
|
||||
private buildQueue(): fastq.queueAsPromised<string, Task<T, R>> {
|
||||
return fastq.promise((id: string) => {
|
||||
const task = this.getTask(id);
|
||||
return this.work(task);
|
||||
}, this.options.concurrency);
|
||||
}
|
||||
|
||||
private async work(task: Task<T, R>): Promise<Task<T, R>> {
|
||||
task.count += 1;
|
||||
task.error = undefined;
|
||||
task.status = 'processing';
|
||||
if (this.options.verbose) {
|
||||
console.log('[task] processing:', task);
|
||||
}
|
||||
try {
|
||||
task.result = await this.worker(task.data);
|
||||
task.status = 'succeeded';
|
||||
if (this.options.verbose) {
|
||||
console.log('[task] succeeded:', task);
|
||||
}
|
||||
return task;
|
||||
} catch (error) {
|
||||
task.error = error;
|
||||
task.status = 'failed';
|
||||
if (this.options.verbose) {
|
||||
console.log('[task] failed:', task);
|
||||
}
|
||||
if (this.options.retry > 0 && task.count < this.options.retry) {
|
||||
if (this.options.verbose) {
|
||||
console.log('[task] retry:', task);
|
||||
}
|
||||
return this.work(task);
|
||||
}
|
||||
return task;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,341 @@
|
||||
import mockfs from 'mock-fs';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { Batcher, CrawlOptions, crawl } from 'src/utils';
|
||||
import { Mock } from 'vitest';
|
||||
|
||||
interface Test {
|
||||
test: string;
|
||||
options: Omit<CrawlOptions, 'extensions'>;
|
||||
files: Record<string, boolean>;
|
||||
skipOnWin32?: boolean;
|
||||
}
|
||||
|
||||
const cwd = process.cwd();
|
||||
|
||||
const readContent = (path: string) => {
|
||||
return readFileSync(path).toString();
|
||||
};
|
||||
|
||||
const extensions = [
|
||||
'.jpg',
|
||||
'.jpeg',
|
||||
'.png',
|
||||
'.heif',
|
||||
'.heic',
|
||||
'.tif',
|
||||
'.nef',
|
||||
'.webp',
|
||||
'.tiff',
|
||||
'.dng',
|
||||
'.gif',
|
||||
'.mov',
|
||||
'.mp4',
|
||||
'.webm',
|
||||
];
|
||||
|
||||
const tests: Test[] = [
|
||||
{
|
||||
test: 'should return empty when crawling an empty path list',
|
||||
options: {
|
||||
pathsToCrawl: [],
|
||||
},
|
||||
files: {},
|
||||
},
|
||||
{
|
||||
test: 'should crawl a single folder',
|
||||
options: {
|
||||
pathsToCrawl: ['/photos/'],
|
||||
},
|
||||
files: {
|
||||
'/photos/image.jpg': true,
|
||||
},
|
||||
},
|
||||
{
|
||||
test: 'should crawl folders with quotes',
|
||||
options: {
|
||||
pathsToCrawl: ["/photo's/", '/photo"s/', '/photo`s/'],
|
||||
},
|
||||
files: {
|
||||
"/photo's/image1.jpg": true,
|
||||
'/photo"s/image2.jpg': true,
|
||||
'/photo`s/image3.jpg': true,
|
||||
},
|
||||
skipOnWin32: true, // single quote interferes with mockfs root on Windows
|
||||
},
|
||||
{
|
||||
test: 'should crawl a single file',
|
||||
options: {
|
||||
pathsToCrawl: ['/photos/image.jpg'],
|
||||
},
|
||||
files: {
|
||||
'/photos/image.jpg': true,
|
||||
},
|
||||
},
|
||||
{
|
||||
test: 'should crawl a single file and a folder',
|
||||
options: {
|
||||
pathsToCrawl: ['/photos/image.jpg', '/images/'],
|
||||
},
|
||||
files: {
|
||||
'/photos/image.jpg': true,
|
||||
'/images/image2.jpg': true,
|
||||
},
|
||||
},
|
||||
{
|
||||
test: 'should exclude by file extension',
|
||||
options: {
|
||||
pathsToCrawl: ['/photos/'],
|
||||
exclusionPattern: '**/*.tif',
|
||||
},
|
||||
files: {
|
||||
'/photos/image.jpg': true,
|
||||
'/photos/image.tif': false,
|
||||
},
|
||||
},
|
||||
{
|
||||
test: 'should exclude by file extension without case sensitivity',
|
||||
options: {
|
||||
pathsToCrawl: ['/photos/'],
|
||||
exclusionPattern: '**/*.TIF',
|
||||
},
|
||||
files: {
|
||||
'/photos/image.jpg': true,
|
||||
'/photos/image.tif': false,
|
||||
},
|
||||
},
|
||||
{
|
||||
test: 'should exclude by folder',
|
||||
options: {
|
||||
pathsToCrawl: ['/photos/'],
|
||||
exclusionPattern: '**/raw/**',
|
||||
recursive: true,
|
||||
},
|
||||
files: {
|
||||
'/photos/image.jpg': true,
|
||||
'/photos/raw/image.jpg': false,
|
||||
'/photos/raw2/image.jpg': true,
|
||||
'/photos/folder/raw/image.jpg': false,
|
||||
'/photos/crawl/image.jpg': true,
|
||||
},
|
||||
},
|
||||
{
|
||||
test: 'should crawl multiple paths',
|
||||
options: {
|
||||
pathsToCrawl: ['/photos/', '/images/', '/albums/'],
|
||||
},
|
||||
files: {
|
||||
'/photos/image1.jpg': true,
|
||||
'/images/image2.jpg': true,
|
||||
'/albums/image3.jpg': true,
|
||||
},
|
||||
},
|
||||
|
||||
{
|
||||
test: 'should crawl a single path without trailing slash',
|
||||
options: {
|
||||
pathsToCrawl: ['/photos'],
|
||||
},
|
||||
files: {
|
||||
'/photos/image.jpg': true,
|
||||
},
|
||||
},
|
||||
{
|
||||
test: 'should crawl a single path',
|
||||
options: {
|
||||
pathsToCrawl: ['/photos/'],
|
||||
recursive: true,
|
||||
},
|
||||
files: {
|
||||
'/photos/image.jpg': true,
|
||||
'/photos/subfolder/image1.jpg': true,
|
||||
'/photos/subfolder/image2.jpg': true,
|
||||
'/image1.jpg': false,
|
||||
},
|
||||
},
|
||||
{
|
||||
test: 'should filter file extensions',
|
||||
options: {
|
||||
pathsToCrawl: ['/photos/'],
|
||||
},
|
||||
files: {
|
||||
'/photos/image.jpg': true,
|
||||
'/photos/image.txt': false,
|
||||
'/photos/1': false,
|
||||
},
|
||||
},
|
||||
{
|
||||
test: 'should include photo and video extensions',
|
||||
options: {
|
||||
pathsToCrawl: ['/photos/', '/videos/'],
|
||||
},
|
||||
files: {
|
||||
'/photos/image.jpg': true,
|
||||
'/photos/image.jpeg': true,
|
||||
'/photos/image.heic': true,
|
||||
'/photos/image.heif': true,
|
||||
'/photos/image.png': true,
|
||||
'/photos/image.gif': true,
|
||||
'/photos/image.tif': true,
|
||||
'/photos/image.tiff': true,
|
||||
'/photos/image.webp': true,
|
||||
'/photos/image.dng': true,
|
||||
'/photos/image.nef': true,
|
||||
'/videos/video.mp4': true,
|
||||
'/videos/video.mov': true,
|
||||
'/videos/video.webm': true,
|
||||
},
|
||||
},
|
||||
{
|
||||
test: 'should check file extensions without case sensitivity',
|
||||
options: {
|
||||
pathsToCrawl: ['/photos/'],
|
||||
},
|
||||
files: {
|
||||
'/photos/image.jpg': true,
|
||||
'/photos/image.Jpg': true,
|
||||
'/photos/image.jpG': true,
|
||||
'/photos/image.JPG': true,
|
||||
'/photos/image.jpEg': true,
|
||||
'/photos/image.TIFF': true,
|
||||
'/photos/image.tif': true,
|
||||
'/photos/image.dng': true,
|
||||
'/photos/image.NEF': true,
|
||||
},
|
||||
},
|
||||
{
|
||||
test: 'should normalize the path',
|
||||
options: {
|
||||
pathsToCrawl: ['/photos/1/../2'],
|
||||
},
|
||||
files: {
|
||||
'/photos/1/image.jpg': false,
|
||||
'/photos/2/image.jpg': true,
|
||||
},
|
||||
},
|
||||
{
|
||||
test: 'should return absolute paths',
|
||||
options: {
|
||||
pathsToCrawl: ['photos'],
|
||||
},
|
||||
files: {
|
||||
[`${cwd}/photos/1.jpg`]: true,
|
||||
[`${cwd}/photos/2.jpg`]: true,
|
||||
[`/photos/3.jpg`]: false,
|
||||
},
|
||||
},
|
||||
{
|
||||
test: 'should support ignoring full filename',
|
||||
options: {
|
||||
pathsToCrawl: ['/photos'],
|
||||
exclusionPattern: '**/image2.jpg',
|
||||
},
|
||||
files: {
|
||||
'/photos/image1.jpg': true,
|
||||
'/photos/image2.jpg': false,
|
||||
'/photos/image3.jpg': true,
|
||||
},
|
||||
},
|
||||
{
|
||||
test: 'should support ignoring file extensions',
|
||||
options: {
|
||||
pathsToCrawl: ['/photos'],
|
||||
exclusionPattern: '**/*.png',
|
||||
},
|
||||
files: {
|
||||
'/photos/image1.jpg': true,
|
||||
'/photos/image2.png': false,
|
||||
'/photos/image3.jpg': true,
|
||||
},
|
||||
},
|
||||
{
|
||||
test: 'should support ignoring folder names',
|
||||
options: {
|
||||
pathsToCrawl: ['/photos'],
|
||||
recursive: true,
|
||||
exclusionPattern: '**/raw/**',
|
||||
},
|
||||
files: {
|
||||
'/photos/image1.jpg': true,
|
||||
'/photos/image/image1.jpg': true,
|
||||
'/photos/raw/image2.dng': false,
|
||||
'/photos/raw/image3.dng': false,
|
||||
'/photos/notraw/image3.jpg': true,
|
||||
},
|
||||
},
|
||||
{
|
||||
test: 'should support ignoring absolute paths',
|
||||
options: {
|
||||
// Currently, fast-glob has some caveat when dealing with `/`.
|
||||
pathsToCrawl: ['/*s'],
|
||||
recursive: true,
|
||||
exclusionPattern: '/images/**',
|
||||
},
|
||||
files: {
|
||||
'/photos/image1.jpg': true,
|
||||
'/images/image2.jpg': false,
|
||||
'/assets/image3.jpg': true,
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
describe('crawl', () => {
|
||||
afterEach(() => {
|
||||
mockfs.restore();
|
||||
});
|
||||
|
||||
describe('crawl', () => {
|
||||
for (const { test: name, options, files, skipOnWin32 } of tests) {
|
||||
if (process.platform === 'win32' && skipOnWin32) {
|
||||
test.skip(name);
|
||||
continue;
|
||||
}
|
||||
it(name, async () => {
|
||||
// The file contents is the same as the path.
|
||||
mockfs(Object.fromEntries(Object.keys(files).map((file) => [file, file])));
|
||||
|
||||
const actual = await crawl({ ...options, extensions });
|
||||
const expected = Object.entries(files)
|
||||
.filter((entry) => entry[1])
|
||||
.map(([file]) => file);
|
||||
|
||||
// Compare file's content instead of path since a file can be represent in multiple ways.
|
||||
expect(actual.map((path) => readContent(path)).toSorted()).toEqual(expected.toSorted());
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('Batcher', () => {
|
||||
let batcher: Batcher;
|
||||
let onBatch: Mock;
|
||||
beforeEach(() => {
|
||||
onBatch = vi.fn();
|
||||
batcher = new Batcher({ batchSize: 2, onBatch });
|
||||
});
|
||||
|
||||
it('should trigger onBatch() when a batch limit is reached', async () => {
|
||||
batcher.add('a');
|
||||
batcher.add('b');
|
||||
batcher.add('c');
|
||||
expect(onBatch).toHaveBeenCalledOnce();
|
||||
expect(onBatch).toHaveBeenCalledWith(['a', 'b']);
|
||||
});
|
||||
|
||||
it('should trigger onBatch() when flush() is called', async () => {
|
||||
batcher.add('a');
|
||||
batcher.flush();
|
||||
expect(onBatch).toHaveBeenCalledOnce();
|
||||
expect(onBatch).toHaveBeenCalledWith(['a']);
|
||||
});
|
||||
|
||||
it('should trigger onBatch() when debounce time reached', async () => {
|
||||
vi.useFakeTimers();
|
||||
batcher = new Batcher({ batchSize: 2, debounceTimeMs: 100, onBatch });
|
||||
batcher.add('a');
|
||||
expect(onBatch).not.toHaveBeenCalled();
|
||||
vi.advanceTimersByTime(200);
|
||||
expect(onBatch).toHaveBeenCalledOnce();
|
||||
expect(onBatch).toHaveBeenCalledWith(['a']);
|
||||
vi.useRealTimers();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,265 @@
|
||||
import { ApiKeyResponseDto, getMyApiKey, getMyUser, init, isHttpError, Permission } from '@immich/sdk';
|
||||
import { convertPathToPattern, glob } from 'fast-glob';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { createReadStream } from 'node:fs';
|
||||
import { readFile, stat, writeFile } from 'node:fs/promises';
|
||||
import { platform } from 'node:os';
|
||||
import { join, resolve } from 'node:path';
|
||||
import yaml from 'yaml';
|
||||
|
||||
export interface BaseOptions {
|
||||
configDirectory: string;
|
||||
key?: string;
|
||||
url?: string;
|
||||
}
|
||||
|
||||
export type AuthDto = { url: string; key: string };
|
||||
type OldAuthDto = { instanceUrl: string; apiKey: string };
|
||||
|
||||
export const authenticate = async (options: BaseOptions): Promise<AuthDto> => {
|
||||
const { configDirectory: configDir, url, key } = options;
|
||||
|
||||
// provided in command
|
||||
if (url && key) {
|
||||
return connect(url, key);
|
||||
}
|
||||
|
||||
// fallback to auth file
|
||||
const config = await readAuthFile(configDir);
|
||||
const auth = await connect(config.url, config.key);
|
||||
if (auth.url !== config.url) {
|
||||
await writeAuthFile(configDir, auth);
|
||||
}
|
||||
|
||||
return auth;
|
||||
};
|
||||
|
||||
export const s = (count: number) => (count === 1 ? '' : 's');
|
||||
|
||||
let _apiKey: ApiKeyResponseDto;
|
||||
export const requirePermissions = async (permissions: Permission[]) => {
|
||||
if (!_apiKey) {
|
||||
_apiKey = await getMyApiKey();
|
||||
}
|
||||
|
||||
if (_apiKey.permissions.includes(Permission.All)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const missing: Permission[] = [];
|
||||
|
||||
for (const permission of permissions) {
|
||||
if (!_apiKey.permissions.includes(permission)) {
|
||||
missing.push(permission);
|
||||
}
|
||||
}
|
||||
|
||||
if (missing.length > 0) {
|
||||
const combined = missing.map((permission) => `"${permission}"`).join(', ');
|
||||
console.log(
|
||||
`Missing required permission${s(missing.length)}: ${combined}.
|
||||
Please make sure your API key has the correct permissions.`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
};
|
||||
|
||||
export const connect = async (url: string, key: string) => {
|
||||
const wellKnownUrl = new URL('.well-known/immich', url);
|
||||
try {
|
||||
const wellKnown = await fetch(wellKnownUrl).then((response) => response.json());
|
||||
const endpoint = new URL(wellKnown.api.endpoint, url).toString();
|
||||
if (endpoint !== url) {
|
||||
console.debug(`Discovered API at ${endpoint}`);
|
||||
}
|
||||
url = endpoint;
|
||||
} catch {
|
||||
// noop
|
||||
}
|
||||
|
||||
init({ baseUrl: url, apiKey: key });
|
||||
|
||||
const [error] = await withError(getMyUser());
|
||||
if (isHttpError(error)) {
|
||||
logError(error, `Failed to connect to server ${url}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
return { url, key };
|
||||
};
|
||||
|
||||
export const logError = (error: unknown, message: string) => {
|
||||
if (isHttpError(error)) {
|
||||
console.error(`${message}: ${error.status}`);
|
||||
console.error(JSON.stringify(error.data, undefined, 2));
|
||||
} else {
|
||||
console.error(`${message} - ${error}`);
|
||||
}
|
||||
};
|
||||
|
||||
export const getAuthFilePath = (dir: string) => join(dir, 'auth.yml');
|
||||
|
||||
export const readAuthFile = async (dir: string) => {
|
||||
try {
|
||||
const data = await readFile(getAuthFilePath(dir));
|
||||
// TODO add class-transform/validation
|
||||
const auth = yaml.parse(data.toString()) as AuthDto | OldAuthDto;
|
||||
const { instanceUrl, apiKey } = auth as OldAuthDto;
|
||||
if (instanceUrl && apiKey) {
|
||||
return { url: instanceUrl, key: apiKey };
|
||||
}
|
||||
return auth as AuthDto;
|
||||
} catch (error: Error | any) {
|
||||
if (error.code === 'ENOENT' || error.code === 'ENOTDIR') {
|
||||
console.log('No auth file exists. Please login first.');
|
||||
process.exit(1);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
export const writeAuthFile = async (dir: string, auth: AuthDto) =>
|
||||
writeFile(getAuthFilePath(dir), yaml.stringify(auth), { mode: 0o600 });
|
||||
|
||||
export const withError = async <T>(promise: Promise<T>): Promise<[Error, undefined] | [undefined, T]> => {
|
||||
try {
|
||||
const result = await promise;
|
||||
return [undefined, result];
|
||||
} catch (error: Error | any) {
|
||||
return [error, undefined];
|
||||
}
|
||||
};
|
||||
|
||||
export interface CrawlOptions {
|
||||
pathsToCrawl: string[];
|
||||
recursive?: boolean;
|
||||
includeHidden?: boolean;
|
||||
exclusionPattern?: string;
|
||||
extensions: string[];
|
||||
}
|
||||
|
||||
const convertPathToPatternOnWin = (path: string) => {
|
||||
return platform() === 'win32' ? convertPathToPattern(path) : path;
|
||||
};
|
||||
|
||||
export const crawl = async (options: CrawlOptions): Promise<string[]> => {
|
||||
const { extensions: extensionsWithPeriod, recursive, pathsToCrawl, exclusionPattern, includeHidden } = options;
|
||||
const extensions = extensionsWithPeriod.map((extension) => extension.replace('.', ''));
|
||||
|
||||
if (pathsToCrawl.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const patterns: string[] = [];
|
||||
const crawledFiles: string[] = [];
|
||||
|
||||
for await (const currentPath of pathsToCrawl) {
|
||||
try {
|
||||
const absolutePath = resolve(currentPath);
|
||||
const stats = await stat(absolutePath);
|
||||
if (stats.isFile() || stats.isSymbolicLink()) {
|
||||
crawledFiles.push(absolutePath);
|
||||
} else {
|
||||
patterns.push(convertPathToPatternOnWin(absolutePath));
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (error.code === 'ENOENT') {
|
||||
patterns.push(convertPathToPatternOnWin(currentPath));
|
||||
} else {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (patterns.length === 0) {
|
||||
return crawledFiles;
|
||||
}
|
||||
|
||||
const searchPatterns = patterns.map((pattern) => {
|
||||
let escapedPattern = pattern.replaceAll("'", "[']").replaceAll('"', '["]').replaceAll('`', '[`]');
|
||||
if (recursive) {
|
||||
escapedPattern = escapedPattern + '/**';
|
||||
}
|
||||
return `${escapedPattern}/*.{${extensions.join(',')}}`;
|
||||
});
|
||||
|
||||
const globbedFiles = await glob(searchPatterns, {
|
||||
absolute: true,
|
||||
caseSensitiveMatch: false,
|
||||
dot: includeHidden,
|
||||
ignore: [`**/${exclusionPattern}`],
|
||||
});
|
||||
globbedFiles.push(...crawledFiles);
|
||||
return globbedFiles.toSorted();
|
||||
};
|
||||
|
||||
export const sha1 = (filepath: string) => {
|
||||
const hash = createHash('sha1');
|
||||
return new Promise<string>((resolve, reject) => {
|
||||
const rs = createReadStream(filepath);
|
||||
rs.on('error', reject);
|
||||
rs.on('data', (chunk) => hash.update(chunk));
|
||||
rs.on('end', () => resolve(hash.digest('hex')));
|
||||
});
|
||||
};
|
||||
|
||||
/**
|
||||
* Batches items and calls onBatch to process them
|
||||
* when the batch size is reached or the debounce time has passed.
|
||||
*/
|
||||
export class Batcher<T = unknown> {
|
||||
private items: T[] = [];
|
||||
private readonly batchSize: number;
|
||||
private readonly debounceTimeMs?: number;
|
||||
private readonly onBatch: (items: T[]) => void;
|
||||
private debounceTimer?: NodeJS.Timeout;
|
||||
|
||||
constructor({
|
||||
batchSize,
|
||||
debounceTimeMs,
|
||||
onBatch,
|
||||
}: {
|
||||
batchSize: number;
|
||||
debounceTimeMs?: number;
|
||||
onBatch: (items: T[]) => Promise<void>;
|
||||
}) {
|
||||
this.batchSize = batchSize;
|
||||
this.debounceTimeMs = debounceTimeMs;
|
||||
this.onBatch = onBatch;
|
||||
}
|
||||
|
||||
private setDebounceTimer() {
|
||||
if (this.debounceTimer) {
|
||||
clearTimeout(this.debounceTimer);
|
||||
}
|
||||
if (this.debounceTimeMs) {
|
||||
this.debounceTimer = setTimeout(() => this.flush(), this.debounceTimeMs);
|
||||
}
|
||||
}
|
||||
|
||||
private clearDebounceTimer() {
|
||||
if (this.debounceTimer) {
|
||||
clearTimeout(this.debounceTimer);
|
||||
this.debounceTimer = undefined;
|
||||
}
|
||||
}
|
||||
|
||||
add(item: T) {
|
||||
this.items.push(item);
|
||||
this.setDebounceTimer();
|
||||
if (this.items.length >= this.batchSize) {
|
||||
this.flush();
|
||||
}
|
||||
}
|
||||
|
||||
flush() {
|
||||
this.clearDebounceTimer();
|
||||
if (this.items.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.onBatch(this.items);
|
||||
|
||||
this.items = [];
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user