84 lines
2.3 KiB
JavaScript
84 lines
2.3 KiB
JavaScript
// api_v1/scanner/scanActivityCartella.js
|
|
const scanActivityFiles = require('./scanActivityFiles');
|
|
const {
|
|
enqueueActivityIngestJobs,
|
|
} = require('./activityIngestQueue');
|
|
const { sha256 } = require('./utils');
|
|
const { log } = require('./logger');
|
|
|
|
const BATCH_SIZE = 100;
|
|
|
|
async function scanActivityCartella(
|
|
db,
|
|
user,
|
|
cartella,
|
|
absCartella,
|
|
newActivities,
|
|
updateStatusFile,
|
|
CURRENT,
|
|
TOTAL_FILES,
|
|
start,
|
|
deleteFromDB,
|
|
onProgress
|
|
) {
|
|
log(`📁 [SCAN ACTIVITY CARTELLA] ${cartella}`);
|
|
|
|
// Se la lettura del filesystem fallisce, lo scan deve fallire:
|
|
// non dobbiamo interpretare tutti i record come orfani.
|
|
const rows = await db('activities')
|
|
.where({ user, cartella })
|
|
.select('id');
|
|
|
|
const idsSet = new Set(rows.map(r => r.id));
|
|
const files = [];
|
|
|
|
// Prima scopriamo tutti i file. Nessun orfano viene cancellato
|
|
// se la scoperta o l'accodamento falliscono.
|
|
for await (const f of scanActivityFiles(user, cartella, absCartella, db)) {
|
|
files.push(f);
|
|
}
|
|
|
|
for (let offset = 0; offset < files.length; offset += BATCH_SIZE) {
|
|
const batch = files.slice(offset, offset + BATCH_SIZE);
|
|
|
|
const jobs = batch.map(f => ({
|
|
user,
|
|
cartella,
|
|
relPath: f.relPath,
|
|
mtimeMs: f.stat.mtimeMs,
|
|
sizeBytes: f.stat.size,
|
|
}));
|
|
|
|
// Una transazione per batch, anziché una per file.
|
|
const results = await enqueueActivityIngestJobs(db, jobs);
|
|
|
|
for (let i = 0; i < batch.length; i++) {
|
|
const f = batch[i];
|
|
const result = results[i];
|
|
|
|
CURRENT.value++;
|
|
idsSet.delete(sha256(`${user}/${cartella}/${f.relPath}`));
|
|
|
|
log(
|
|
`📥 [ACTIVITY QUEUE] ${f.relPath} ` +
|
|
`status=${result.status}` +
|
|
`${result.inserted ? ' inserted' : ''}` +
|
|
`${result.updated ? ' updated' : ''}` +
|
|
`${result.unchanged ? ' unchanged' : ''}`
|
|
);
|
|
}
|
|
|
|
// Aggiorna l'interfaccia una volta per batch, non per file.
|
|
await updateStatusFile(CURRENT, TOTAL_FILES, start);
|
|
if (onProgress) await onProgress();
|
|
}
|
|
|
|
// Questa sezione viene raggiunta soltanto se tutti i batch
|
|
// sono stati accodati correttamente.
|
|
for (const orphanId of idsSet) {
|
|
log(`🔴 [ORPHAN ACTIVITY] ${orphanId}`);
|
|
await deleteFromDB(orphanId, user);
|
|
}
|
|
}
|
|
|
|
module.exports = scanActivityCartella;
|