feat(ai): scorecards de QA, avaliacao automatica, dashboard (fase 21)

CRUD de QualityScorecard/Item (criterios por tenant, sem lista fixa
hardcoded), novo AIJobType.SCORECARD_EVALUATION encadeado junto com
ANALYSIS a partir da transcricao (mesma decisao de privacidade + exige
scorecard habilitado). apps/ai-worker avalia contra todos os scorecards
habilitados do tenant, prompt montado dinamicamente a partir dos itens de
cada um, nunca guarda chain-of-thought do modelo (so' o resultado final
validado). GET /reports/ai-dashboard agrega CallAIAnalysis+QualityEvaluation
do periodo (score medio, sentimento, assuntos/objecoes, compliance
alerts, ranking de agentes).

Testado ponta a ponta contra o ai-worker real e Postgres real com RLS
(scorecard real via API, prompt montado a partir dos itens reais, job
real reservado via SKIP LOCKED, retry+dead-letter corretos) — chamada de
rede real contra OpenAI/Anthropic continua nunca exercitada. Detalhes em
docs/QUALITY_SCORECARDS.md.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01X1HxY46WGU4G1zmVDNKcWw

apps/ai-worker/src/claim.ts

@@ -3,7 +3,7 @@ import type { Prisma } from "@b2bcall/database";
 export interface ClaimedJob {
   id: string;
   callId: string;
-  type: "TRANSCRIPTION" | "ANALYSIS" | "REANALYSIS";
+  type: "TRANSCRIPTION" | "ANALYSIS" | "REANALYSIS" | "SCORECARD_EVALUATION";
   attemptCount: number;
   maxAttempts: number;
 }

apps/ai-worker/src/main.ts

@@ -3,6 +3,7 @@ import { createLogger } from "@b2bcall/shared";
 import { claimPendingJobs, type ClaimedJob } from "./claim";
 import { processTranscriptionJob } from "./process-transcription";
 import { processAnalysisJob } from "./process-analysis";
+import { processScorecardJob } from "./process-scorecard";
 import { markJobCompleted, markJobFailed } from "./job-outcome";
 
 const logger = createLogger("b2bcall-ai-worker");
@@ -17,6 +18,8 @@ async function processJob(tenantId: string, job: ClaimedJob): Promise<void> {
       await processTranscriptionJob(tenantId, job.callId);
     } else if (job.type === "ANALYSIS" || job.type === "REANALYSIS") {
       await processAnalysisJob(tenantId, job.callId);
+    } else if (job.type === "SCORECARD_EVALUATION") {
+      await processScorecardJob(tenantId, job.callId);
     } else {
       throw new Error(`Tipo de AIJob desconhecido: ${job.type}`);
     }

apps/ai-worker/src/process-scorecard.ts

@@ -0,0 +1,120 @@
+import { getPrismaClient, withTenantContext } from "@b2bcall/database";
+import {
+  SensitiveDataRedactor,
+  QUALITY_EVALUATION_JSON_SCHEMA,
+  validateQualityEvaluationResult,
+} from "@b2bcall/ai";
+import { createLogger } from "@b2bcall/shared";
+import { resolveProviderForCapability } from "./provider-resolution";
+
+const logger = createLogger("b2bcall-ai-worker");
+const redactor = new SensitiveDataRedactor();
+
+/** Monta o prompt a partir dos itens do scorecard (secao 117: weight,
+ * description, evaluation_prompt são por tenant, nunca hardcoded aqui) —
+ * o schema de resposta só define a FORMA (score + mapa de
+ * criterionScores), quem diz QUAIS critérios usar como chave é este
+ * texto. */
+function buildScorecardPrompt(scorecardName: string, items: { name: string; weight: number; description: string | null; evaluationPrompt: string | null }[]): string {
+  const itemLines = items
+    .map((item) => {
+      const parts = [`- "${item.name}" (peso ${item.weight})`];
+      if (item.description) parts.push(item.description);
+      if (item.evaluationPrompt) parts.push(`Como avaliar: ${item.evaluationPrompt}`);
+      return parts.join(" — ");
+    })
+    .join("\n");
+
+  return [
+    `Avalie a qualidade desta ligação de atendimento contra o scorecard "${scorecardName}".`,
+    `Critérios (a chave em criterionScores deve ser exatamente o nome entre aspas de cada um):`,
+    itemLines,
+    `Dê um score de 0 a 100 pra cada critério em "criterionScores", um "score" geral de 0 a 100 (pode considerar os pesos), e um "summaryJustification" curto. Nunca inclua seu raciocínio passo a passo, só o resultado final.`,
+  ].join("\n\n");
+}
+
+/**
+ * QA automático (agente.md secao 118): avalia a chamada contra TODOS os
+ * `QualityScorecard` habilitados do tenant — uma `QualityEvaluation` por
+ * scorecard, não só o primeiro. Se o tenant não tiver nenhum scorecard
+ * habilitado, o job não deveria nem ter sido criado (checado em
+ * `process-transcription.ts` antes de encadear); chegar aqui sem nenhum é
+ * tratado como falha (config mudou entre os dois momentos), não como
+ * sucesso silencioso.
+ */
+export async function processScorecardJob(tenantId: string, callId: string): Promise<void> {
+  const prisma = getPrismaClient();
+
+  const { transcription, scorecards } = await withTenantContext(prisma, tenantId, async (tx) => {
+    const transcription = await tx.callTranscription.findFirst({
+      where: { callId, status: "COMPLETED" },
+      orderBy: { createdAt: "desc" },
+    });
+    const scorecards = await tx.qualityScorecard.findMany({
+      where: { tenantId, enabled: true },
+      include: { items: true },
+    });
+    return { transcription, scorecards };
+  });
+
+  if (!transcription?.text) {
+    throw new Error(`Nenhuma transcricao concluida encontrada pra chamada ${callId}`);
+  }
+  if (scorecards.length === 0) {
+    throw new Error("Nenhum QualityScorecard habilitado pra este tenant (config mudou depois do encadeamento)");
+  }
+
+  const resolved = await withTenantContext(prisma, tenantId, (tx) =>
+    resolveProviderForCapability(tx, tenantId, "STRUCTURED_OUTPUT"),
+  );
+  if (!resolved) {
+    throw new Error("Nenhum provider/modelo com capability STRUCTURED_OUTPUT habilitado pra este tenant");
+  }
+  if (!resolved.instance.analyze) {
+    throw new Error(`Provider ${resolved.providerId} nao implementa analyze()`);
+  }
+
+  const redactedText = redactor.redact(transcription.text);
+
+  for (const scorecard of scorecards) {
+    const promptContent = buildScorecardPrompt(scorecard.name, scorecard.items);
+
+    const result = await resolved.instance.analyze({
+      transcriptText: redactedText,
+      promptContent,
+      jsonSchema: QUALITY_EVALUATION_JSON_SCHEMA,
+    });
+    const validated = validateQualityEvaluationResult(result.data);
+
+    await withTenantContext(prisma, tenantId, async (tx) => {
+      await tx.qualityEvaluation.create({
+        data: {
+          tenantId,
+          callId,
+          scorecardId: scorecard.id,
+          score: validated.score,
+          criterionScores: validated.criterionScores,
+          summaryJustification: validated.summaryJustification,
+        },
+      });
+
+      const usageRows = [
+        { type: "AI_ANALYSIS_REQUEST" as const, quantity: 1 },
+        ...(result.inputTokens ? [{ type: "AI_INPUT_TOKENS" as const, quantity: result.inputTokens }] : []),
+        ...(result.outputTokens ? [{ type: "AI_OUTPUT_TOKENS" as const, quantity: result.outputTokens }] : []),
+      ];
+      await tx.aIUsageRecord.createMany({
+        data: usageRows.map((row) => ({
+          tenantId,
+          callId,
+          type: row.type,
+          quantity: row.quantity,
+          providerId: resolved.providerId,
+          model: resolved.externalModelId,
+        })),
+      });
+    });
+
+    logger.info("avaliacao de QA concluida", { callId, scorecardId: scorecard.id, score: validated.score });
+  }
+}

apps/ai-worker/src/process-transcription.ts

@@ -132,6 +132,20 @@ export async function processTranscriptionJob(tenantId: string, callId: string):
         tx.aIJob.create({ data: { tenantId, callId, type: "ANALYSIS" } }),
       );
       logger.info("job de analise de IA encadeado", { callId });
+
+      // QA automatico (secao 118) tambem depende de analise estar
+      // autorizada (usa o mesmo texto redigido, mesma decisao de
+      // privacidade) — só encadeia se o tenant tiver algum scorecard
+      // habilitado, senao o job falharia certo de cara.
+      const hasEnabledScorecard = await withTenantContext(prisma, tenantId, (tx) =>
+        tx.qualityScorecard.count({ where: { tenantId, enabled: true } }),
+      );
+      if (hasEnabledScorecard > 0) {
+        await withTenantContext(prisma, tenantId, (tx) =>
+          tx.aIJob.create({ data: { tenantId, callId, type: "SCORECARD_EVALUATION" } }),
+        );
+        logger.info("job de avaliacao de QA encadeado", { callId });
+      }
     }
   } finally {
     await cleanupJobWorkDir(workDir);

apps/api/src/app.module.ts

@@ -16,6 +16,7 @@ import { CallsModule } from "./calls/calls.module";
 import { ReportsModule } from "./reports/reports.module";
 import { RecordingsModule } from "./recordings/recordings.module";
 import { AIModule } from "./ai/ai.module";
+import { QualityModule } from "./quality/quality.module";
 
 @Module({
   imports: [
@@ -36,6 +37,7 @@ import { AIModule } from "./ai/ai.module";
     ReportsModule,
     RecordingsModule,
     AIModule,
+    QualityModule,
   ],
 })
 export class AppModule {}

apps/api/src/quality/dto/create-quality-scorecard.dto.ts

@@ -0,0 +1,31 @@
+import { IsArray, IsNumber, IsOptional, IsString, MaxLength, ValidateNested } from "class-validator";
+import { Type } from "class-transformer";
+
+export class CreateQualityScorecardItemDto {
+  @IsString()
+  @MaxLength(120)
+  name!: string;
+
+  @IsOptional()
+  @IsNumber()
+  weight?: number;
+
+  @IsOptional()
+  @IsString()
+  description?: string;
+
+  @IsOptional()
+  @IsString()
+  evaluationPrompt?: string;
+}
+
+export class CreateQualityScorecardDto {
+  @IsString()
+  @MaxLength(120)
+  name!: string;
+
+  @IsArray()
+  @ValidateNested({ each: true })
+  @Type(() => CreateQualityScorecardItemDto)
+  items!: CreateQualityScorecardItemDto[];
+}

apps/api/src/quality/quality-scorecards.controller.ts

@@ -0,0 +1,89 @@
+import { Controller, Body, Delete, Get, HttpCode, HttpStatus, NotFoundException, Param, Post, UseGuards } from "@nestjs/common";
+import { getPrismaClient, withTenantContext } from "@b2bcall/database";
+import { recordAuditEvent, type AccessTokenClaims } from "@b2bcall/auth";
+import { JwtAuthGuard } from "../common/guards/jwt-auth.guard";
+import { PermissionGuard } from "../common/guards/permission.guard";
+import { RequirePermission } from "../common/decorators/require-permission.decorator";
+import { CurrentUser } from "../common/decorators/current-user.decorator";
+import { CreateQualityScorecardDto } from "./dto/create-quality-scorecard.dto";
+
+/**
+ * Scorecards de QA (agente.md secao 117-118) — critérios ponderados
+ * (`weight`) por tenant, sem lista fixa hardcoded (secao 117 sugere
+ * Saudação/Identificação/Empatia/etc. como exemplo, não como enum). Cada
+ * chamada avaliada vira uma `QualityEvaluation` por scorecard habilitado
+ * (`apps/ai-worker/src/process-scorecard.ts`), nunca editando um
+ * scorecard "quebra" avaliações já feitas (`QualityEvaluation` guarda o
+ * resultado, não referencia os items em si).
+ */
+@UseGuards(JwtAuthGuard, PermissionGuard)
+@Controller("quality/scorecards")
+export class QualityScorecardsController {
+  @RequirePermission("ai.manage")
+  @Post()
+  async create(@CurrentUser() user: AccessTokenClaims, @Body() dto: CreateQualityScorecardDto) {
+    const prisma = getPrismaClient();
+    const tenantId = user.tenantId!;
+
+    const scorecard = await withTenantContext(prisma, tenantId, (tx) =>
+      tx.qualityScorecard.create({
+        data: {
+          tenantId,
+          name: dto.name,
+          items: {
+            create: dto.items.map((item) => ({
+              tenantId,
+              name: item.name,
+              weight: item.weight ?? 1,
+              description: item.description,
+              evaluationPrompt: item.evaluationPrompt,
+            })),
+          },
+        },
+        include: { items: true },
+      }),
+    );
+
+    await recordAuditEvent(prisma, {
+      action: "QUALITY_SCORECARD_CREATE",
+      tenantId,
+      userId: user.sub,
+      entityType: "quality_scorecard",
+      entityId: scorecard.id,
+      after: { name: scorecard.name, itemCount: scorecard.items.length },
+    });
+
+    return scorecard;
+  }
+
+  @RequirePermission("ai.view")
+  @Get()
+  async list(@CurrentUser() user: AccessTokenClaims) {
+    const prisma = getPrismaClient();
+    const tenantId = user.tenantId!;
+    return withTenantContext(prisma, tenantId, (tx) =>
+      tx.qualityScorecard.findMany({ where: { enabled: true }, include: { items: true }, orderBy: { name: "asc" } }),
+    );
+  }
+
+  @RequirePermission("ai.manage")
+  @Delete(":id")
+  @HttpCode(HttpStatus.NO_CONTENT)
+  async remove(@CurrentUser() user: AccessTokenClaims, @Param("id") id: string) {
+    const prisma = getPrismaClient();
+    const tenantId = user.tenantId!;
+
+    const result = await withTenantContext(prisma, tenantId, (tx) =>
+      tx.qualityScorecard.updateMany({ where: { id, tenantId }, data: { enabled: false } }),
+    );
+    if (result.count === 0) throw new NotFoundException();
+
+    await recordAuditEvent(prisma, {
+      action: "QUALITY_SCORECARD_DELETE",
+      tenantId,
+      userId: user.sub,
+      entityType: "quality_scorecard",
+      entityId: id,
+    });
+  }
+}

apps/api/src/quality/quality.module.ts

@@ -0,0 +1,7 @@
+import { Module } from "@nestjs/common";
+import { QualityScorecardsController } from "./quality-scorecards.controller";
+
+@Module({
+  controllers: [QualityScorecardsController],
+})
+export class QualityModule {}

apps/api/src/reports/reports.controller.ts

@@ -18,6 +18,15 @@ function average(values: number[]): number | null {
   return values.reduce((sum, v) => sum + v, 0) / values.length;
 }
 
+function topCounts(lists: string[][], limit: number): { value: string; count: number }[] {
+  const counts = new Map<string, number>();
+  for (const list of lists) for (const value of list) counts.set(value, (counts.get(value) ?? 0) + 1);
+  return Array.from(counts.entries())
+    .sort((a, b) => b[1] - a[1])
+    .slice(0, limit)
+    .map(([value, count]) => ({ value, count }));
+}
+
 @UseGuards(JwtAuthGuard, PermissionGuard)
 @Controller("reports")
 export class ReportsController {
@@ -200,4 +209,69 @@ export class ReportsController {
       }),
     );
   }
+
+  /** Dashboard de IA (secao 119). `scoreMedio` = média de
+   * `CallAIAnalysis.qualityScore` (avaliação livre do modelo sobre a
+   * chamada); `avgScorecardScore` = média de `QualityEvaluation.score`
+   * (avaliação formal contra um scorecard, secao 117-118) — dois números
+   * diferentes, a especificação só pede "score médio" sem dizer qual dos
+   * dois, então mostra os dois em vez de escolher um. */
+  @RequirePermission("reports.view")
+  @Get("ai-dashboard")
+  async aiDashboard(@CurrentUser() user: AccessTokenClaims, @Query("from") from?: string, @Query("to") to?: string) {
+    const prisma = getPrismaClient();
+    const tenantId = user.tenantId!;
+    const range = dateRange(from, to);
+
+    const [analyses, evaluations] = await withTenantContext(prisma, tenantId, (tx) =>
+      Promise.all([
+        tx.callAIAnalysis.findMany({
+          where: { tenantId, createdAt: range },
+          include: { call: { select: { agentId: true } } },
+        }),
+        tx.qualityEvaluation.findMany({
+          where: { tenantId, createdAt: range },
+          include: { call: { select: { agentId: true } } },
+        }),
+      ]),
+    );
+
+    const agentIds = Array.from(
+      new Set([...analyses, ...evaluations].map((r) => r.call.agentId).filter((v): v is string => v != null)),
+    );
+    const agents = agentIds.length
+      ? await withTenantContext(prisma, tenantId, (tx) =>
+          tx.agent.findMany({ where: { id: { in: agentIds } }, select: { id: true, name: true } }),
+        )
+      : [];
+    const agentName = (id: string) => agents.find((a) => a.id === id)?.name ?? id;
+
+    const sentimentCounts: Record<string, number> = {};
+    for (const a of analyses) if (a.sentiment) sentimentCounts[a.sentiment] = (sentimentCounts[a.sentiment] ?? 0) + 1;
+
+    const agentScores = new Map<string, number[]>();
+    for (const a of analyses) {
+      if (!a.call.agentId || a.agentScore == null) continue;
+      const list = agentScores.get(a.call.agentId) ?? [];
+      list.push(a.agentScore);
+      agentScores.set(a.call.agentId, list);
+    }
+    const agentRanking = Array.from(agentScores.entries())
+      .map(([agentId, scores]) => ({ agentId, name: agentName(agentId), avgScore: average(scores)!, calls: scores.length }))
+      .sort((x, y) => y.avgScore - x.avgScore);
+
+    return {
+      callsAnalyzed: analyses.length,
+      avgQualityScore: average(analyses.map((a) => a.qualityScore).filter((v): v is number => v != null)),
+      avgScorecardScore: average(evaluations.map((e) => e.score)),
+      sentimentBreakdown: sentimentCounts,
+      topTopics: topCounts(analyses.map((a) => a.topics), 10),
+      topObjections: topCounts(analyses.map((a) => a.objections), 10),
+      complianceAlerts: analyses
+        .filter((a) => a.complianceFlags.length > 0)
+        .map((a) => ({ callId: a.callId, flags: a.complianceFlags })),
+      topAgents: agentRanking.slice(0, 5),
+      bottomAgents: agentRanking.slice(-5).reverse(),
+    };
+  }
 }