feat(import): bulk tool import (CSV/JSON/YAML) for admins; NetBox-style help button
Build & Push Docker Image / build (push) Successful in 2m49s
Build & Push Docker Image / build (push) Successful in 2m49s
- Add POST /admin/tools/import with format auto-detect, CSV delimiters
(comma/semicolon/tab), per-row validation via CreateToolBody, bulk insert,
audit log entries per imported tool; gated by new 'tool-import' feature
flag (premium/enterprise; admins always pass)
- Add tool-import-dialog UI (format tabs, delimiter select, textarea, file
upload, result/error list) behind hasFeature('tool-import')
- Replace FieldHelp question marks and bare GuideHelp links with a NetBox-style
'Hilfe/Help' outline button (HelpCircle + text) in form headers only
- Sync locales to 482 keys per language (de/en), update handbook docs
(administration import section, index/plaene feature tables), regenerate
API client + zod schemas, add yaml dependency
This commit is contained in:
@@ -23,6 +23,7 @@
|
||||
"openid-client": "6.8.4",
|
||||
"pino": "10.3.1",
|
||||
"pino-http": "11.0.0",
|
||||
"yaml": "catalog:",
|
||||
"zod": "catalog:"
|
||||
},
|
||||
"devDependencies": {
|
||||
|
||||
@@ -2,8 +2,8 @@ import { type Request, type Response, type NextFunction } from "express";
|
||||
|
||||
const TIER_FEATURES: Record<string, string[]> = {
|
||||
free: ["browse", "rate", "search"],
|
||||
premium: ["browse", "rate", "search", "similar-tools", "costs", "redundancy", "analytics-advanced", "trash", "compare", "watchlist"],
|
||||
enterprise: ["browse", "rate", "search", "similar-tools", "costs", "redundancy", "analytics-advanced", "trash", "compare", "watchlist", "sso", "audit-export", "api-access"],
|
||||
premium: ["browse", "rate", "search", "similar-tools", "costs", "redundancy", "analytics-advanced", "trash", "compare", "watchlist", "tool-import"],
|
||||
enterprise: ["browse", "rate", "search", "similar-tools", "costs", "redundancy", "analytics-advanced", "trash", "compare", "watchlist", "tool-import", "sso", "audit-export", "api-access"],
|
||||
};
|
||||
|
||||
export function getEntitlements(tier: string | undefined, role: string | undefined): string[] {
|
||||
|
||||
@@ -1,9 +1,12 @@
|
||||
import { Router, type IRouter } from "express";
|
||||
import { eq, and, sql } from "drizzle-orm";
|
||||
import { z } from "zod";
|
||||
import { parse as parseYaml } from "yaml";
|
||||
import { db, toolsTable, ratingsTable, toolCostsTable, toolRelationsTable } from "@workspace/db";
|
||||
import { requireAdmin } from "../middleware/auth";
|
||||
import { requireFeature } from "../middleware/feature";
|
||||
import { writeAuditLog } from "../lib/audit";
|
||||
import { CreateToolBody } from "@workspace/api-zod";
|
||||
|
||||
const EvaluateBody = z.object({
|
||||
toolId: z.coerce.number().int().positive(),
|
||||
@@ -219,4 +222,184 @@ router.post("/admin/redundancy/evaluate", requireAdmin, async (req, res): Promis
|
||||
res.json({ ok: true });
|
||||
});
|
||||
|
||||
const ToolImportBody = z.object({
|
||||
format: z.enum(["auto", "csv", "json", "yaml"]).default("auto"),
|
||||
delimiter: z.enum(["auto", "comma", "semicolon", "tab"]).default("auto"),
|
||||
data: z.string().min(1),
|
||||
});
|
||||
|
||||
const TOOL_FIELDS = ["name", "description", "category", "websiteUrl", "iconUrl", "features", "tags"] as const;
|
||||
|
||||
function splitList(value: unknown): string[] {
|
||||
if (Array.isArray(value)) {
|
||||
return value.map((v) => String(v).trim()).filter(Boolean);
|
||||
}
|
||||
if (typeof value === "string") {
|
||||
return value
|
||||
.split(/[|;,]/)
|
||||
.map((v) => v.trim())
|
||||
.filter(Boolean);
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
function normalizeTool(raw: Record<string, unknown>): Record<string, unknown> {
|
||||
const out: Record<string, unknown> = {};
|
||||
for (const key of TOOL_FIELDS) {
|
||||
if (key in raw) out[key] = raw[key];
|
||||
}
|
||||
if ("features" in out) out.features = splitList(out.features);
|
||||
if ("tags" in out) out.tags = splitList(out.tags);
|
||||
return out;
|
||||
}
|
||||
|
||||
function parseCsv(data: string, delimiter: string): Record<string, unknown>[] {
|
||||
let sep = ",";
|
||||
if (delimiter === "semicolon") sep = ";";
|
||||
else if (delimiter === "tab") sep = "\t";
|
||||
else if (delimiter === "auto") {
|
||||
const firstLine = data.split(/\r?\n/)[0] ?? "";
|
||||
const candidates = [",", ";", "\t"];
|
||||
const counts = candidates.map((c) => (firstLine.match(new RegExp(`\\${c === "\t" ? "\\t" : c}`, "g")) ?? []).length);
|
||||
const max = Math.max(...counts);
|
||||
if (max > 0) sep = candidates[counts.indexOf(max)];
|
||||
}
|
||||
|
||||
const lines = data.split(/\r?\n/).filter((l) => l.trim().length > 0);
|
||||
if (lines.length === 0) return [];
|
||||
const header = parseCsvLine(lines[0], sep);
|
||||
const rows: Record<string, unknown>[] = [];
|
||||
for (let i = 1; i < lines.length; i++) {
|
||||
const cells = parseCsvLine(lines[i], sep);
|
||||
const row: Record<string, unknown> = {};
|
||||
header.forEach((h, idx) => {
|
||||
const key = h.trim();
|
||||
if (key) row[key] = cells[idx] ?? "";
|
||||
});
|
||||
rows.push(row);
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
function parseCsvLine(line: string, sep: string): string[] {
|
||||
const cells: string[] = [];
|
||||
let cur = "";
|
||||
let inQuotes = false;
|
||||
for (let i = 0; i < line.length; i++) {
|
||||
const ch = line[i];
|
||||
if (inQuotes) {
|
||||
if (ch === '"') {
|
||||
if (line[i + 1] === '"') {
|
||||
cur += '"';
|
||||
i++;
|
||||
} else {
|
||||
inQuotes = false;
|
||||
}
|
||||
} else {
|
||||
cur += ch;
|
||||
}
|
||||
} else if (ch === '"') {
|
||||
inQuotes = true;
|
||||
} else if (ch === sep) {
|
||||
cells.push(cur);
|
||||
cur = "";
|
||||
} else {
|
||||
cur += ch;
|
||||
}
|
||||
}
|
||||
cells.push(cur);
|
||||
return cells;
|
||||
}
|
||||
|
||||
router.post(
|
||||
"/admin/tools/import",
|
||||
requireAdmin,
|
||||
requireFeature("tool-import"),
|
||||
async (req, res): Promise<void> => {
|
||||
const parsed = ToolImportBody.safeParse(req.body);
|
||||
if (!parsed.success) {
|
||||
res.status(400).json({ error: parsed.error.message });
|
||||
return;
|
||||
}
|
||||
const { format, delimiter, data } = parsed.data;
|
||||
|
||||
let rawRows: Record<string, unknown>[] = [];
|
||||
let effectiveFormat = format;
|
||||
if (format === "auto") {
|
||||
const trimmed = data.trimStart();
|
||||
if (trimmed.startsWith("[") || trimmed.startsWith("{")) {
|
||||
effectiveFormat = "json";
|
||||
} else if (trimmed.includes(":") && !trimmed.startsWith("\"")) {
|
||||
effectiveFormat = "yaml";
|
||||
} else {
|
||||
effectiveFormat = "csv";
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
if (effectiveFormat === "json") {
|
||||
const json = JSON.parse(data);
|
||||
if (Array.isArray(json)) {
|
||||
rawRows = json.filter((r) => r && typeof r === "object") as Record<string, unknown>[];
|
||||
} else if (json && typeof json === "object") {
|
||||
rawRows = [json as Record<string, unknown>];
|
||||
} else {
|
||||
res.status(400).json({ error: "JSON data must be an object or an array of objects" });
|
||||
return;
|
||||
}
|
||||
} else if (effectiveFormat === "yaml") {
|
||||
const yaml = parseYaml(data);
|
||||
if (Array.isArray(yaml)) {
|
||||
rawRows = yaml.filter((r) => r && typeof r === "object") as Record<string, unknown>[];
|
||||
} else if (yaml && typeof yaml === "object") {
|
||||
rawRows = [yaml as Record<string, unknown>];
|
||||
} else {
|
||||
res.status(400).json({ error: "YAML data must be an object or a list of objects" });
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
rawRows = parseCsv(data, delimiter);
|
||||
}
|
||||
} catch (err) {
|
||||
res.status(400).json({ error: `Failed to parse ${effectiveFormat} data: ${(err as Error).message}` });
|
||||
return;
|
||||
}
|
||||
|
||||
const errors: { row: number; error: string }[] = [];
|
||||
const validRows: { name: string; description: string; category: string; websiteUrl?: string; iconUrl?: string; features: string[]; tags: string[]; createdBy: string }[] = [];
|
||||
const user = req.session.user!;
|
||||
const createdBy = user.preferred_username || user.name || user.sub;
|
||||
|
||||
rawRows.forEach((raw, idx) => {
|
||||
const normalized = normalizeTool(raw);
|
||||
const check = CreateToolBody.safeParse(normalized);
|
||||
if (!check.success) {
|
||||
errors.push({ row: idx + 1, error: check.error.message });
|
||||
return;
|
||||
}
|
||||
validRows.push({
|
||||
name: check.data.name,
|
||||
description: check.data.description,
|
||||
category: check.data.category,
|
||||
websiteUrl: check.data.websiteUrl,
|
||||
iconUrl: check.data.iconUrl,
|
||||
features: check.data.features ?? [],
|
||||
tags: check.data.tags ?? [],
|
||||
createdBy,
|
||||
});
|
||||
});
|
||||
|
||||
let imported = 0;
|
||||
if (validRows.length > 0) {
|
||||
const inserted = await db.insert(toolsTable).values(validRows).returning({ id: toolsTable.id, name: toolsTable.name, category: toolsTable.category });
|
||||
imported = inserted.length;
|
||||
for (const tool of inserted) {
|
||||
await writeAuditLog(req, "tool", tool.id, "create", { name: tool.name, category: tool.category, source: "import" });
|
||||
}
|
||||
}
|
||||
|
||||
res.json({ imported, total: rawRows.length, errors });
|
||||
},
|
||||
);
|
||||
|
||||
export default router;
|
||||
|
||||
Reference in New Issue
Block a user