misc: sds format support, convert limits, dell dedup, supermicro removal, bible updates

Parser / archive: - Add .sds extension as tar-format alias (archive.go) - Add tests for multipart upload size limits (multipart_limits_test.go) - Remove supermicro crashdump parser (ADL-015) Dell parser: - Remove GPU duplicates from PCIeDevices (DCIM_VideoView vs DCIM_PCIDeviceView both list the same GPU; VideoView record is authoritative) Server: - Add LOGPILE_CONVERT_MAX_MB env var for independent convert batch size limit - Improve "file too large" error message with current limit value Web: - Add CONVERT_MAX_FILES_PER_BATCH = 1000 cap - Minor UI copy and CSS fixes Bible: - bible-local/06-parsers.md: add pci.ids enrichment rule (enrich model from pciids when name is empty but vendor_id+device_id are present) - Sync bible submodule and local overview/architecture docs Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-03-01 22:23:44 +03:00
parent 9c5512d238
commit 21ea129933
22 changed files with 268 additions and 446 deletions
--- a/web/static/css/style.css
+++ b/web/static/css/style.css
@@ -384,38 +384,43 @@ main {

 .parsers-title {
    font-size: 0.85rem;
-    color: #666;
-    margin-bottom: 0.5rem;
+    color: #4b5563;
+    margin-bottom: 0.6rem;
+    font-weight: 600;
 }

 .parsers-list {
    display: flex;
    flex-wrap: wrap;
-    gap: 0.5rem;
+    gap: 0.6rem;
    justify-content: center;
 }

-.parser-item {
+.parser-chip {
    display: inline-flex;
    align-items: center;
-    gap: 0.5rem;
-    background: #f8f9fa;
-    padding: 0.4rem 0.8rem;
-    border-radius: 4px;
-    border: 1px solid #e0e0e0;
+    gap: 0.45rem;
+    background: #eef6ff;
+    padding: 0.38rem 0.72rem;
+    border-radius: 999px;
+    border: 1px solid #bfdcff;
+    line-height: 1;
 }

-.parser-name {
+.parser-chip-name {
    font-size: 0.85rem;
-    color: #2c3e50;
+    color: #1f2937;
+    font-weight: 500;
 }

-.parser-version {
-    font-size: 0.75rem;
-    color: #888;
-    background: #e8e8e8;
-    padding: 0.1rem 0.4rem;
-    border-radius: 3px;
+.parser-chip-version {
+    font-size: 0.72rem;
+    color: #1d4ed8;
+    background: #dbeafe;
+    padding: 0.12rem 0.42rem;
+    border-radius: 999px;
+    border: 1px solid #bfdbfe;
+    font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace;
 }

 /* File Info */
--- a/web/static/js/app.js
+++ b/web/static/js/app.js
@@ -14,6 +14,7 @@ document.addEventListener('DOMContentLoaded', () => {
 let sourceType = 'archive';
 let convertFiles = [];
 let isConvertRunning = false;
+const CONVERT_MAX_FILES_PER_BATCH = 1000;
 let supportedUploadExtensions = null;
 let supportedConvertExtensions = null;
 let apiConnectPayload = null;
@@ -539,12 +540,12 @@ async function loadParsersInfo() {
        const container = document.getElementById('parsers-info');

        if (data.parsers && data.parsers.length > 0) {
-            let html = '<p class="parsers-title">Поддерживаемые платформы:</p><div class="parsers-list">';
+            let html = '<p class="parsers-title">Подключенные парсеры:</p><div class="parsers-list">';
            data.parsers.forEach(p => {
-                html += `<div class="parser-item">
-                    <span class="parser-name">${escapeHtml(p.name)}</span>
-                    <span class="parser-version">v${escapeHtml(p.version)}</span>
-                </div>`;
+                html += `<span class="parser-chip">
+                    <span class="parser-chip-name">${escapeHtml(p.name)}</span>
+                    <span class="parser-chip-version">v${escapeHtml(p.version)}</span>
+                </span>`;
            });
            html += '</div>';
            container.innerHTML = html;
@@ -653,8 +654,10 @@ function renderConvertSummary() {
    const remaining = supportedFiles.length - previewFiles.length;
    const previewText = previewFiles.length > 0 ? `Примеры: ${previewFiles.join(', ')}` : '';
    const skippedText = skippedCount > 0 ? ` Пропущено неподдерживаемых: ${skippedCount}.` : '';
+    const batchCount = Math.ceil(supportedFiles.length / CONVERT_MAX_FILES_PER_BATCH);
+    const batchesText = batchCount > 1 ? ` Будет ${batchCount} прохода(ов) по ${CONVERT_MAX_FILES_PER_BATCH} файлов.` : '';

-    summary.innerHTML = `<strong>${supportedFiles.length}</strong> файлов готовы к конвертации.${previewText ? ` ${previewText}` : ''}${remaining > 0 ? ` и ещё ${remaining}` : ''}.${skippedText}`;
+    summary.innerHTML = `<strong>${supportedFiles.length}</strong> файлов готовы к конвертации.${previewText ? ` ${previewText}` : ''}${remaining > 0 ? ` и ещё ${remaining}` : ''}.${skippedText}${batchesText}`;
    summary.className = 'api-connect-status';
 }

@@ -674,58 +677,71 @@ async function runConvertBatch() {
        renderConvertStatus('В выбранной папке нет файлов поддерживаемого типа', 'error');
        return;
    }
+    const batches = chunkFiles(supportedFiles, CONVERT_MAX_FILES_PER_BATCH);

    isConvertRunning = true;
    runButton.disabled = true;
    renderConvertProgress(0, 'Подготовка загрузки...');
-    renderConvertStatus('Выполняю пакетную конвертацию...', 'info');
-
-    const formData = new FormData();
-    supportedFiles.forEach(file => {
-        const relativePath = file.webkitRelativePath || file.name || 'file';
-        formData.append('files[]', file, relativePath);
-    });
+    renderConvertStatus(`Выполняю пакетную конвертацию (${batches.length} проходов)...`, 'info');

    try {
-        const startResponse = await uploadConvertBatch(formData, (percent) => {
-            const uploadPercent = Math.round(percent * 0.3);
-            renderConvertProgress(uploadPercent, `Загрузка файлов: ${percent}%`);
-        });
-
-        if (!startResponse.ok) {
-            const errorPayload = parseConvertErrorPayload(startResponse.bodyText);
-            hideConvertProgress();
-            renderConvertStatus(errorPayload.error || 'Пакетная конвертация завершилась с ошибкой', 'error');
-            return;
-        }
-
-        if (!startResponse.jobId) {
-            hideConvertProgress();
-            renderConvertStatus('Сервер не вернул идентификатор задачи', 'error');
-            return;
-        }
-
-        await waitForConvertJob(startResponse.jobId, (statusPayload) => {
-            const serverProgress = Number(statusPayload.progress || 0);
-            const combined = 30 + Math.round(Math.max(0, Math.min(100, serverProgress)) * 0.7);
-            renderConvertProgress(combined, `Конвертация: ${serverProgress}%`);
-        });
-
-        renderConvertProgress(100, 'Подготовка выгрузки...');
-        const downloadResponse = await downloadConvertArchive(startResponse.jobId);
-        if (!downloadResponse.ok) {
-            const errorPayload = parseConvertErrorPayload(downloadResponse.bodyText);
-            hideConvertProgress();
-            renderConvertStatus(errorPayload.error || 'Не удалось скачать результат', 'error');
-            return;
-        }
-
-        const blob = downloadResponse.blob;
        const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
-        downloadBlob(blob, `logpile-convert-${timestamp}.zip`);
-        const summary = downloadResponse.summaryHeader || 'Конвертация завершена';
+        const passSummaries = [];
+
+        for (let batchIdx = 0; batchIdx < batches.length; batchIdx++) {
+            const batchFiles = batches[batchIdx];
+            const pass = batchIdx + 1;
+            const passLabel = `Проход ${pass}/${batches.length}`;
+            const passStart = Math.round((batchIdx / batches.length) * 100);
+            const passEnd = Math.round(((batchIdx + 1) / batches.length) * 100);
+
+            const formData = new FormData();
+            batchFiles.forEach(file => {
+                const relativePath = file.webkitRelativePath || file.name || 'file';
+                formData.append('files[]', file, relativePath);
+            });
+
+            const startResponse = await uploadConvertBatch(formData, (percent) => {
+                const clamped = Math.max(0, Math.min(100, Number(percent) || 0));
+                const uploadPhase = passStart + Math.round((passEnd - passStart) * 0.3 * (clamped / 100));
+                renderConvertProgress(uploadPhase, `${passLabel}: загрузка ${clamped}%`);
+            });
+
+            if (!startResponse.ok) {
+                const errorPayload = parseConvertErrorPayload(startResponse.bodyText);
+                hideConvertProgress();
+                renderConvertStatus(`${passLabel}: ${errorPayload.error || 'пакетная конвертация завершилась с ошибкой'}`, 'error');
+                return;
+            }
+
+            if (!startResponse.jobId) {
+                hideConvertProgress();
+                renderConvertStatus(`${passLabel}: сервер не вернул идентификатор задачи`, 'error');
+                return;
+            }
+
+            await waitForConvertJob(startResponse.jobId, (statusPayload) => {
+                const serverProgress = Math.max(0, Math.min(100, Number(statusPayload.progress || 0)));
+                const phase = 0.3 + 0.7 * (serverProgress / 100);
+                const combined = passStart + Math.round((passEnd - passStart) * phase);
+                renderConvertProgress(combined, `${passLabel}: конвертация ${serverProgress}%`);
+            });
+
+            const downloadResponse = await downloadConvertArchive(startResponse.jobId);
+            if (!downloadResponse.ok) {
+                const errorPayload = parseConvertErrorPayload(downloadResponse.bodyText);
+                hideConvertProgress();
+                renderConvertStatus(`${passLabel}: ${errorPayload.error || 'не удалось скачать результат'}`, 'error');
+                return;
+            }
+
+            const suffix = batches.length > 1 ? `-part${pass}` : '';
+            downloadBlob(downloadResponse.blob, `logpile-convert-${timestamp}${suffix}.zip`);
+            passSummaries.push(downloadResponse.summaryHeader || `${passLabel}: завершено`);
+        }
+
        hideConvertProgress();
-        renderConvertStatus(summary, 'success');
+        renderConvertStatus(passSummaries.join(' | '), 'success');
    } catch (err) {
        hideConvertProgress();
        renderConvertStatus('Ошибка соединения при конвертации', 'error');
@@ -735,6 +751,15 @@ async function runConvertBatch() {
    }
 }

+function chunkFiles(files, chunkSize) {
+    const safeChunkSize = Math.max(1, Number(chunkSize) || 1);
+    const chunks = [];
+    for (let i = 0; i < files.length; i += safeChunkSize) {
+        chunks.push(files.slice(i, i + safeChunkSize));
+    }
+    return chunks;
+}
+
 function uploadConvertBatch(formData, onUploadPercent) {
    return new Promise((resolve, reject) => {
        const xhr = new XMLHttpRequest();