src/platform/config-import/config-import.service.ts
No results matching.
Properties |
|
Methods |
|
constructor(configService: ConfigService, migrationService?: ConfigMigrationService, ownershipService?: ConfigOwnershipService, resourceRegistry?: ConfigResourceRegistry, importModeService?: ConfigImportModeService)
|
||||||||||||||||||
|
Parameters :
|
| extractErrorMessages | ||||||
extractErrorMessages(error: unknown)
|
||||||
|
Extract nested error messages from validation errors
Parameters :
Returns :
string[]
|
| formatNestedValidationError | ||||||
formatNestedValidationError(error: unknown)
|
||||||
|
Parameters :
Returns :
string
|
| Async importConfigs | ||||||
| Use importConfigsForTenant with the orchestrator's tenant-by-tenant approach instead. | ||||||
importConfigs<T>(options: ImportOptions<T>)
|
||||||
Type parameters :
|
||||||
|
Generic import method that handles the common pattern across all services.
Parameters :
Returns :
Promise<void>
|
| Async importConfigsForTenant | |||||||||
importConfigsForTenant<T>(tenantId: string, options: TenantImportOptions<T>)
|
|||||||||
Type parameters :
|
|||||||||
|
Import configs for a specific tenant. This is the preferred method when using the orchestrator's tenant-by-tenant approach.
Parameters :
Returns :
Promise<void>
|
| Private Async markFileManaged | ||||||||||||||||||
markFileManaged(tenantId: string, kind: unknown, document: ConfigDocument, filePath: string, raw: string)
|
||||||||||||||||||
|
Parameters :
Returns :
Promise<void>
|
| Private resolveMode |
resolveMode()
|
|
Returns :
"disabled" | unknown
|
| Async validateConfig | ||||||||||||||||||||||||
validateConfig<T>(filePath: string, file: string, payload: any, schemaOrDto: any, tenant: { name: string }, resourceType: string, formatError?: (error?: unknown) => void)
|
||||||||||||||||||||||||
Type parameters :
|
||||||||||||||||||||||||
|
Validate configuration against a Zod schema or parse-capable DTO.
Parameters :
Returns :
Promise<{ isValid: boolean; data: T }>
|
| Private Readonly logger |
Type : unknown
|
Default value : new Logger(ConfigImportService.name)
|
import { createHash } from "node:crypto";
import { existsSync, readdirSync, readFileSync } from "node:fs";
import { join } from "node:path";
import { Injectable, Logger, Optional } from "@nestjs/common";
import { ConfigService } from "@nestjs/config";
import { ConfigMigrationService } from "../config-portability/config-migration.service";
import { ConfigOwnershipService } from "../config-portability/config-ownership.service";
import { ConfigResourceRegistry } from "../config-portability/config-resource.registry";
import type { ConfigDocument } from "../config-portability/config-resource.types";
import { ConfigImportModeService } from "./config-import-mode.service";
import { ImportOptions, TenantImportOptions } from "./import-options";
function resolveValidationSchema(schemaOrDto: unknown): any | undefined {
if (!schemaOrDto) {
return undefined;
}
const candidate = schemaOrDto as any;
if (typeof candidate.safeParse === "function") {
return candidate;
}
if (typeof candidate.schema?.safeParse === "function") {
return candidate.schema;
}
return undefined;
}
@Injectable()
export class ConfigImportService {
private readonly logger = new Logger(ConfigImportService.name);
constructor(
private readonly configService: ConfigService,
@Optional()
private readonly migrationService?: ConfigMigrationService,
@Optional()
private readonly ownershipService?: ConfigOwnershipService,
@Optional()
private readonly resourceRegistry?: ConfigResourceRegistry,
@Optional()
private readonly importModeService?: ConfigImportModeService,
) {}
/**
* Import configs for a specific tenant.
* This is the preferred method when using the orchestrator's tenant-by-tenant approach.
*/
async importConfigsForTenant<T extends object>(
tenantId: string,
options: TenantImportOptions<T>,
): Promise<void> {
const configPath = this.configService.getOrThrow("CONFIG_FOLDER");
const mode = this.resolveMode();
if (mode === "disabled") return;
const updateExisting = mode !== "create";
const strictConfig = this.configService.get<any>(
"CONFIG_VARIABLE_STRICT",
);
let counter = 0;
const path = join(configPath, tenantId, options.subfolder);
if (!existsSync(path)) {
return;
}
const files = readdirSync(path);
for (const file of files) {
const filePath = join(path, file);
// Filter by extension if provided
if (
options.fileExtension &&
!file.endsWith(options.fileExtension)
) {
continue;
}
try {
// Load data using custom loader or default JSON loader
let data: T;
let portableDocument: ConfigDocument | undefined;
const resourceKind =
options.resourceKind ??
this.resourceRegistry?.inferKind(options.resourceType);
const raw = readFileSync(filePath, "utf8");
if (
resourceKind &&
this.migrationService &&
file.endsWith(".json")
) {
const payload = JSON.parse(raw) as Record<string, unknown>;
const fileId = file.replace(/\.json$/i, "");
const wrapped = this.migrationService.isDocument(payload)
? payload
: this.migrationService.wrapLegacy(
resourceKind,
payload,
String(payload.id ?? payload.clientId ?? fileId),
);
const upgraded = this.migrationService.upgrade(wrapped);
const blocking = upgraded.issues.filter(
(issue) => issue.severity !== "warning",
);
if (blocking.length > 0) {
throw new Error(
`Configuration migration requires input: ${blocking
.map(
(issue) =>
`${issue.path}: ${issue.message}`,
)
.join("; ")}`,
);
}
portableDocument = upgraded.document;
data = this.migrationService.unwrapForLegacyImporter(
upgraded.document,
) as T;
} else if (options.loadData) {
data = await Promise.resolve(options.loadData(filePath));
} else {
const payload = JSON.parse(raw);
data = payload as T;
}
// Replace placeholders like ${ENV_VAR} or ${ENV_VAR:default}
data = this.replacePlaceholders(data);
// Validate if validation schema is provided
const schemaOrDto =
options.validationSchema ?? options.validationClass;
if (schemaOrDto) {
const validationResult = await this.validateConfig(
filePath,
file,
data,
schemaOrDto,
{ name: tenantId },
options.resourceType,
options.formatValidationError,
);
if (!validationResult.isValid) {
continue; // Skip invalid config
}
data = validationResult.data as T;
}
// Check if exists
const exists = await options
.checkExists(tenantId, data, file)
.catch(() => false);
if (
exists &&
portableDocument &&
resourceKind &&
this.ownershipService
) {
const stored = await this.ownershipService.get(
tenantId,
resourceKind,
portableDocument.metadata.id,
);
const incomingGeneration =
portableDocument.metadata.generation ?? 1;
if (incomingGeneration < stored.generation) {
throw new Error(
`Stale configuration generation ${incomingGeneration}; stored generation is ${stored.generation}`,
);
}
}
if (exists && !updateExisting) {
this.logger.debug(
`[${tenantId}] ${options.resourceType} ${file} already exists, skipping`,
);
continue;
}
// Delete existing if force is enabled
if (exists && updateExisting && options.deleteExisting) {
await options.deleteExisting(tenantId, data, file);
}
// Process and store item
await options.processItem(tenantId, data, file);
if (portableDocument && resourceKind && this.ownershipService) {
await this.markFileManaged(
tenantId,
resourceKind,
portableDocument,
filePath,
raw,
);
}
counter++;
} catch (error: any) {
const reason = error?.message || "Unknown error";
this.logger.error(
`[${tenantId}] Failed to import ${options.resourceType} ${file} (${filePath}): ${reason}`,
);
if (strictConfig === "abort") {
// Abort the entire import process in strict abort mode
throw error;
}
}
}
if (counter > 0) {
this.logger.log(
`[${tenantId}] ${counter} ${options.resourceType}(s) imported`,
);
}
}
private async markFileManaged(
tenantId: string,
kind: import("../config-portability/config-resource.types").ConfigResourceKind,
document: ConfigDocument,
filePath: string,
raw: string,
): Promise<void> {
await this.ownershipService!.markApplied({
tenantId,
kind,
resourceId: document.metadata.id,
ownership: "file-managed",
generation: document.metadata.generation ?? 1,
source: filePath,
sourceHash: createHash("sha256").update(raw).digest("hex"),
});
}
/**
* Generic import method that handles the common pattern across all services.
* @deprecated Use importConfigsForTenant with the orchestrator's tenant-by-tenant approach instead.
*/
async importConfigs<T extends object>(
options: ImportOptions<T>,
): Promise<void> {
const mode = this.resolveMode();
if (mode === "disabled") {
return;
}
const configPath = this.configService.getOrThrow("CONFIG_FOLDER");
const updateExisting = mode !== "create";
const tenantFolders = readdirSync(configPath, {
withFileTypes: true,
}).filter((tenant) => tenant.isDirectory());
const strictConfig = this.configService.get<any>(
"CONFIG_VARIABLE_STRICT",
);
for (const tenant of tenantFolders) {
let counter = 0;
const path = join(configPath, tenant.name, options.subfolder);
if (!existsSync(path)) {
continue;
}
const files = readdirSync(path);
for (const file of files) {
const filePath = join(path, file);
// Filter by extension if provided
if (
options.fileExtension &&
!file.endsWith(options.fileExtension)
) {
continue;
}
try {
// Load data using custom loader or default JSON loader
let data: T;
if (options.loadData) {
data = await Promise.resolve(
options.loadData(filePath),
);
} else {
const payload = JSON.parse(
readFileSync(filePath, "utf8"),
);
data = payload as T;
}
// Replace placeholders like ${ENV_VAR} or ${ENV_VAR:default}
data = this.replacePlaceholders(data);
// Validate if validation class is provided
const schemaOrDto =
options.validationSchema ?? options.validationClass;
if (schemaOrDto) {
const validationResult = await this.validateConfig(
filePath,
file,
data,
schemaOrDto,
tenant,
options.resourceType,
options.formatValidationError,
);
if (!validationResult.isValid) {
continue; // Skip invalid config
}
data = validationResult.data as T;
}
// Check if exists
const exists = await options
.checkExists(tenant.name, data, file)
.catch(() => false);
if (exists && !updateExisting) {
this.logger.debug(
`[${tenant.name}] ${options.resourceType} ${file} already exists, skipping`,
);
continue;
}
// Delete existing if force is enabled
if (exists && updateExisting && options.deleteExisting) {
await options.deleteExisting(tenant.name, data, file);
}
// Process and store item
await options.processItem(tenant.name, data, file);
counter++;
} catch (error: any) {
const reason = error?.message || "Unknown error";
this.logger.error(
`[${tenant.name}] Failed to import ${options.resourceType} ${file} (${filePath}): ${reason}`,
);
if (strictConfig === "abort") {
// Abort the entire import process in strict abort mode
throw error;
}
}
}
if (counter > 0) {
this.logger.log(
`[${tenant.name}] ${counter} ${options.resourceType}(s) imported`,
);
}
}
}
/**
* Recursively replace placeholders of the form ${VAR} or ${VAR:default} in all string properties.
* ${VAR} -> replaced with process.env.VAR if defined; if undefined and no default given, logs a warning and leaves placeholder intact.
* ${VAR:default} -> replaced with env value if defined, otherwise with "default".
*/
replacePlaceholders<T>(input: T): T {
const seen = new WeakSet();
const isObject = (val: any) =>
val && typeof val === "object" && !Array.isArray(val);
const strictConfigInner = this.configService.get<any>(
"CONFIG_VARIABLE_STRICT",
);
const strictMode =
strictConfigInner === true
? "skip"
: strictConfigInner === false || strictConfigInner === undefined
? "ignore"
: (strictConfigInner as string);
const processString = (str: string): string => {
const pattern = /\$\{([A-Z0-9_]+)(?::([^}]*))?\}/g;
return str.replaceAll(
pattern,
(fullMatch, varName: string, defVal: string) => {
const envVal = process.env[varName];
if (envVal !== undefined && envVal !== "") {
return envVal;
}
if (defVal !== undefined) {
return defVal;
}
if (
strictMode === "abort" ||
strictMode === "skip" ||
strictMode === "true"
) {
// abort -> will bubble up and stop the whole process via outer catch
// skip/true -> outer catch will log and continue with next file
throw new Error(
`Missing required environment variable ${varName} for placeholder ${fullMatch}`,
);
}
// ignore/false/undefined: keep placeholder and warn
this.logger.warn(
`Environment variable ${varName} not set and no default provided (placeholder kept)`,
);
return fullMatch; // keep original placeholder
},
);
};
const recurse = (val: any): any => {
if (typeof val === "string") return processString(val);
if (Array.isArray(val)) return val.map(recurse);
if (Buffer.isBuffer(val)) return val; // skip binary
if (isObject(val)) {
if (seen.has(val)) return val; // avoid circular refs
seen.add(val);
for (const key of Object.keys(val)) {
val[key] = recurse(val[key]);
}
return val;
}
return val;
};
return recurse(input);
}
private resolveMode():
| "disabled"
| import("../config-portability/config-resource.types").ConfigImportMode {
if (this.importModeService) {
return this.importModeService.resolve();
}
if (!this.configService.get<boolean>("CONFIG_IMPORT")) {
return "disabled";
}
return this.configService.get<boolean>("CONFIG_IMPORT_FORCE")
? "upsert"
: "create";
}
/**
* Validate configuration against a Zod schema or parse-capable DTO.
*/
async validateConfig<T extends object>(
filePath: string,
file: string,
payload: any,
schemaOrDto: any,
tenant: { name: string },
resourceType: string,
formatError?: (error: unknown) => any,
): Promise<{ isValid: boolean; data: T }> {
const schema = resolveValidationSchema(schemaOrDto);
if (!schema) {
throw new Error(
`Validation requested for ${resourceType} ${file} (${filePath}) but no Zod schema was provided`,
);
}
const parsed = schema.safeParse(payload) as
| { success: true; data: T }
| { success: false; error: any };
if (!parsed.success) {
const formatter =
formatError ||
((error: unknown) => {
if (
error &&
typeof error === "object" &&
"path" in error &&
"message" in error
) {
return error;
}
return { message: "Invalid config" };
});
this.logger.error(
{ errors: parsed.error.issues.map(formatter) },
`[${tenant.name}] Validation failed for ${resourceType} ${file} (${filePath})`,
);
return { isValid: false, data: payload };
}
return { isValid: true, data: parsed.data as T };
}
/**
* Extract nested error messages from validation errors
*/
extractErrorMessages(error: unknown): string[] {
if (
error &&
typeof error === "object" &&
"message" in error &&
typeof (error as { message?: unknown }).message === "string"
) {
return [(error as { message: string }).message];
}
return ["Invalid config"];
}
formatNestedValidationError(error: unknown): string {
return this.extractErrorMessages(error).join(", ");
}
}