Skip to content

Y
yaoai-video
Commit 2ce18b88 authored by yaoke.yk's avatar yaoke.yk
Browse files
Options

Agent制作fix6

parent b29011ef
Hide whitespace changes
Inline Side-by-side
Showing with 792 additions and 8 deletions
doc/html/src/app/pages/StoryboardWorkspace.tsx
View file @ 2ce18b88
...@@ -95,6 +95,21 @@ function characterFrontUrl(c: Character): string | null { ...@@ -95,6 +95,21 @@ function characterFrontUrl(c: Character): string | null {
return c.frontImageUrl ?? c.imageUrl ?? null; return c.frontImageUrl ?? c.imageUrl ?? null;
} }
function refTokens(value?: string | null): string[] {
if (!value) return [];
return value.split(",").map((token) => token.trim()).filter(Boolean);
}
function inferSceneRefFromCharacters(characters: string | null | undefined, scenes: Scene[]): string | null {
const sceneRefs = new Set(
scenes
.map((scene) => scene.name?.trim())
.filter(Boolean)
.map((name) => `@${name}`)
);
return refTokens(characters).find((token) => sceneRefs.has(token)) ?? null;
}
export function StoryboardWorkspace() { export function StoryboardWorkspace() {
const { episodeId: urlEpisodeId, projectId } = useParams<{ projectId: string; episodeId?: string }>(); const { episodeId: urlEpisodeId, projectId } = useParams<{ projectId: string; episodeId?: string }>();
const navigate = useNavigate(); const navigate = useNavigate();
...@@ -238,7 +253,7 @@ export function StoryboardWorkspace() { ...@@ -238,7 +253,7 @@ export function StoryboardWorkspace() {
setCharacterKeys(initialChars); setCharacterKeys(initialChars);
setSceneKey(lastTask?.sceneImageKey ?? ""); setSceneKey(lastTask?.sceneImageKey ?? "");
setPropKeys(lastTask?.propImageKeys ?? []); setPropKeys(lastTask?.propImageKeys ?? []);
setFreeformPrompt(lastTask?.userPrompt?.freeformPrompt ?? ""); setFreeformPrompt(expanded.videoPrompt?.trim() || (lastTask?.userPrompt?.freeformPrompt ?? ""));
if (typeof lastTask?.videoDuration === "number") setSelectedDuration(lastTask.videoDuration); if (typeof lastTask?.videoDuration === "number") setSelectedDuration(lastTask.videoDuration);
if (lastTask?.videoRatio) setSelectedRatio(lastTask.videoRatio); if (lastTask?.videoRatio) setSelectedRatio(lastTask.videoRatio);
if (typeof lastTask?.generateAudio === "boolean") setAudioOn(lastTask.generateAudio); if (typeof lastTask?.generateAudio === "boolean") setAudioOn(lastTask.generateAudio);
...@@ -373,6 +388,20 @@ export function StoryboardWorkspace() { ...@@ -373,6 +388,20 @@ export function StoryboardWorkspace() {
const previewTask = previewSb ? getTaskForSb(previewSb) : null; const previewTask = previewSb ? getTaskForSb(previewSb) : null;
const promptPreviewText = freeformPrompt.trim(); const promptPreviewText = freeformPrompt.trim();
const sceneRefDisplay = expanded
? expanded.sceneRef?.trim() || inferSceneRefFromCharacters(expanded.characters, scenes) || "未标注"
: "";
const charactersDisplay = expanded?.characters?.trim() || "未标注";
const startFrameDisplay = expanded?.startFramePrompt?.trim() || "未生成";
const motionScriptDisplay = expanded?.motionScript?.trim() || "未生成";
const videoPromptDisplay = expanded?.videoPrompt?.trim() || "未生成";
const storyboardInfoFields = expanded ? [
{ label: "出镜角色", value: charactersDisplay },
{ label: "分镜场景", value: sceneRefDisplay },
{ label: "首帧提示词", value: startFrameDisplay },
{ label: "镜头运动", value: motionScriptDisplay },
{ label: "视频提示词", value: videoPromptDisplay, multiline: true },
] : [];
return ( return (
<div className="h-full flex overflow-hidden bg-[#f4f7fb] text-foreground"> <div className="h-full flex overflow-hidden bg-[#f4f7fb] text-foreground">
...@@ -431,7 +460,7 @@ export function StoryboardWorkspace() { ...@@ -431,7 +460,7 @@ export function StoryboardWorkspace() {
<span className="absolute left-0 top-3 bottom-3 w-[3px] bg-[#1c64ff]/55 rounded-r-full" /> <span className="absolute left-0 top-3 bottom-3 w-[3px] bg-[#1c64ff]/55 rounded-r-full" />
<div className="flex items-center justify-between mb-1.5"> <div className="flex items-center justify-between mb-1.5">
<span className="text-[11px] font-semibold tracking-[0.12em] uppercase text-muted-foreground">分镜备注</span> <span className="text-[11px] font-semibold tracking-[0.12em] uppercase text-muted-foreground">分镜备注</span>
{promptPreviewText && ( {(promptPreviewText || expanded) && (
<button <button
onClick={() => setShowPromptDetails(true)} onClick={() => setShowPromptDetails(true)}
className="text-[10px] text-[#1c64ff] hover:text-[#164fd0] flex items-center gap-0.5 font-semibold transition-colors" className="text-[10px] text-[#1c64ff] hover:text-[#164fd0] flex items-center gap-0.5 font-semibold transition-colors"
...@@ -789,6 +818,7 @@ export function StoryboardWorkspace() { ...@@ -789,6 +818,7 @@ export function StoryboardWorkspace() {
{showPromptDetails && ( {showPromptDetails && (
<PromptDetailsModal <PromptDetailsModal
text={freeformPrompt} text={freeformPrompt}
agentInfo={storyboardInfoFields}
onClose={() => setShowPromptDetails(false)} onClose={() => setShowPromptDetails(false)}
/> />
)} )}
...@@ -799,6 +829,36 @@ export function StoryboardWorkspace() { ...@@ -799,6 +829,36 @@ export function StoryboardWorkspace() {
// ──────────────────────────────────────────────────────────────────────────── // ────────────────────────────────────────────────────────────────────────────
// StoryboardCard — accordion item with self-contained editor when expanded // StoryboardCard — accordion item with self-contained editor when expanded
// ──────────────────────────────────────────────────────────────────────────── // ────────────────────────────────────────────────────────────────────────────
function StoryboardInfoField({
label,
value,
multiline = false,
compact = false,
}: {
label: string;
value: string;
multiline?: boolean;
compact?: boolean;
}) {
return (
<div className={compact ? "px-3 py-2" : "px-3 py-2.5"}>
<div className="flex items-start gap-2">
<span className="w-16 flex-shrink-0 text-[11px] font-medium text-muted-foreground leading-relaxed">
{label}
</span>
<span
className={`min-w-0 flex-1 text-[11px] leading-relaxed text-foreground/85 ${
multiline ? "line-clamp-3 break-words" : "truncate"
}`}
title={value}
>
{value}
</span>
</div>
</div>
);
}
interface StoryboardCardProps { interface StoryboardCardProps {
sb: Storyboard; sb: Storyboard;
isExpanded: boolean; isExpanded: boolean;
...@@ -1483,9 +1543,10 @@ function AddCharacterModal({ projectId, onClose, onAdded }: AddCharacterModalPro ...@@ -1483,9 +1543,10 @@ function AddCharacterModal({ projectId, onClose, onAdded }: AddCharacterModalPro
// ──────────────────────────────────────────────────────────────────────────── // ────────────────────────────────────────────────────────────────────────────
interface PromptDetailsModalProps { interface PromptDetailsModalProps {
text: string; text: string;
agentInfo: Array<{ label: string; value: string; multiline?: boolean }>;
onClose: () => void; onClose: () => void;
} }
function PromptDetailsModal({ text, onClose }: PromptDetailsModalProps) { function PromptDetailsModal({ text, agentInfo, onClose }: PromptDetailsModalProps) {
const trimmed = text.trim(); const trimmed = text.trim();
return ( return (
<div className="fixed inset-0 bg-slate-950/55 backdrop-blur-sm z-50 flex items-center justify-center p-4" onClick={onClose}> <div className="fixed inset-0 bg-slate-950/55 backdrop-blur-sm z-50 flex items-center justify-center p-4" onClick={onClose}>
...@@ -1499,7 +1560,7 @@ function PromptDetailsModal({ text, onClose }: PromptDetailsModalProps) { ...@@ -1499,7 +1560,7 @@ function PromptDetailsModal({ text, onClose }: PromptDetailsModalProps) {
<X className="w-4 h-4 text-muted-foreground" /> <X className="w-4 h-4 text-muted-foreground" />
</button> </button>
</div> </div>
<div className="flex-1 overflow-y-auto p-5"> <div className="flex-1 overflow-y-auto p-5 space-y-4">
{trimmed ? ( {trimmed ? (
<pre className="text-sm text-foreground leading-relaxed whitespace-pre-wrap font-mono rounded-xl border border-slate-200/80 bg-[#f8fafc] p-4"> <pre className="text-sm text-foreground leading-relaxed whitespace-pre-wrap font-mono rounded-xl border border-slate-200/80 bg-[#f8fafc] p-4">
{trimmed} {trimmed}
...@@ -1507,6 +1568,22 @@ function PromptDetailsModal({ text, onClose }: PromptDetailsModalProps) { ...@@ -1507,6 +1568,22 @@ function PromptDetailsModal({ text, onClose }: PromptDetailsModalProps) {
) : ( ) : (
<p className="text-sm text-muted-foreground italic">（未填写提示词）</p> <p className="text-sm text-muted-foreground italic">（未填写提示词）</p>
)} )}
{agentInfo.length > 0 && (
<div className="rounded-xl border border-slate-200/80 bg-white divide-y divide-slate-100 overflow-hidden">
<div className="px-3 py-2 flex items-center justify-between bg-[#f8fafc]">
<span className="text-xs font-medium text-foreground">Agent 分镜信息</span>
<span className="text-[10px] text-muted-foreground">仅作参考</span>
</div>
{agentInfo.map((item) => (
<StoryboardInfoField
key={item.label}
label={item.label}
value={item.value}
multiline={item.multiline}
/>
))}
</div>
)}
</div> </div>
</div> </div>
</div> </div>
......
doc/html/src/lib/api/ai.ts
View file @ 2ce18b88
...@@ -31,6 +31,8 @@ export interface Storyboard { ...@@ -31,6 +31,8 @@ export interface Storyboard {
shortDescription: string; shortDescription: string;
detailedDescription: string; detailedDescription: string;
characters: string; characters: string;
sceneRef?: string | null;
videoPrompt?: string | null;
dialogues: string; dialogues: string;
cameraDirection: string; cameraDirection: string;
compositionGuide: string; compositionGuide: string;
......
docs/superpowers/plans/2026-04-28-agent-storyboard-enrichment.md 0 → 100644
View file @ 2ce18b88
# Agent Storyboard Enrichment Implementation Plan
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
**Goal:** Make Agent-generated storyboards persist and expose `sceneRef` and `videoPrompt`, while keeping the existing script-import and storyboard generation flow compatible.
**Architecture:** Add two columns to `storyboards`, normalize storyboard generation output so scene references are stored separately from character references, then extend the existing prompt-generation pipeline to persist `video_prompt`. Agent `storyboard_gen` keeps the same outward step shape, but internally runs “generate storyboards -> fill missing video prompts” and exposes richer progress/log output. Frontend changes are limited to type updates and storyboard workspace display.
**Tech Stack:** Spring Boot, MyBatis-Plus, Flyway, React, TypeScript, TanStack Query, Maven
---
## File Structure
**Files to create**
- `yaoai-comic-studio/yaoai-bootstrap/src/main/resources/db/migration/V17__storyboard_scene_ref_video_prompt.sql`
- `yaoai-comic-studio/yaoai-bootstrap/src/test/java/com/yaoai/pipeline/service/impl/StoryboardPipelineServiceImplTest.java`
- `yaoai-comic-studio/yaoai-bootstrap/src/test/java/com/yaoai/agent/service/impl/AgentRunServiceImplTest.java`
**Files to modify**
- `yaoai-comic-studio/yaoai-domain/src/main/java/com/yaoai/domain/entity/Storyboard.java`
- `yaoai-comic-studio/yaoai-domain/src/main/java/com/yaoai/domain/mapper/StoryboardMapper.java`
- `yaoai-comic-studio/yaoai-api/src/main/java/com/yaoai/api/dto/ai/StoryboardDTO.java`
- `yaoai-comic-studio/yaoai-api/src/main/java/com/yaoai/api/controller/StoryboardController.java`
- `yaoai-comic-studio/yaoai-pipeline/src/main/java/com/yaoai/pipeline/service/StoryboardPipelineService.java`
- `yaoai-comic-studio/yaoai-pipeline/src/main/java/com/yaoai/pipeline/service/impl/StoryboardPipelineServiceImpl.java`
- `yaoai-comic-studio/yaoai-agent/src/main/java/com/yaoai/agent/service/impl/AgentRunServiceImpl.java`
- `doc/html/src/lib/api/ai.ts`
- `doc/html/src/hooks/useAi.ts`
- `doc/html/src/app/pages/StoryboardWorkspace.tsx`
**Primary boundaries**
- Database and DTO changes stay centered on the `storyboards` aggregate.
- Prompt persistence stays inside `StoryboardPipelineServiceImpl`; controller and Agent only orchestrate.
- Frontend uses the existing storyboard list API and does not require a new route.
---
### Task 1: Add storyboard storage contract
**Files:**
- Create: `yaoai-comic-studio/yaoai-bootstrap/src/main/resources/db/migration/V17__storyboard_scene_ref_video_prompt.sql`
- Modify: `yaoai-comic-studio/yaoai-domain/src/main/java/com/yaoai/domain/entity/Storyboard.java`
- Modify: `yaoai-comic-studio/yaoai-domain/src/main/java/com/yaoai/domain/mapper/StoryboardMapper.java`
- Modify: `yaoai-comic-studio/yaoai-api/src/main/java/com/yaoai/api/dto/ai/StoryboardDTO.java`
- Test: `yaoai-comic-studio/yaoai-bootstrap/src/test/java/com/yaoai/pipeline/service/impl/StoryboardPipelineServiceImplTest.java`
- [ ] **Step 1: Write the migration and DTO contract test**
```java
@Test
void from_shouldExposeSceneRefAndVideoPrompt() {
Storyboard sb = new Storyboard();
sb.setId(1L);
sb.setSceneRef("@咖啡馆");
sb.setVideoPrompt("0-3s：中景推进，咖啡馆内安静对视");
StoryboardDTO dto = StoryboardDTO.from(sb);
assertEquals("@咖啡馆", dto.getSceneRef());
assertEquals("0-3s：中景推进，咖啡馆内安静对视", dto.getVideoPrompt());
}
```
- [ ] **Step 2: Run test to verify it fails**
Run:
```bash
mvn -q -pl yaoai-comic-studio/yaoai-bootstrap -am -Dtest=StoryboardPipelineServiceImplTest#from_shouldExposeSceneRefAndVideoPrompt test
```
Expected: FAIL because `sceneRef` and `videoPrompt` fields do not exist yet.
- [ ] **Step 3: Add Flyway migration**
```sql
ALTER TABLE storyboards
ADD COLUMN scene_ref VARCHAR(128) DEFAULT NULL
COMMENT '分镜主场景引用，格式 @场景名',
ADD COLUMN video_prompt LONGTEXT DEFAULT NULL
COMMENT '分镜视频生成提示词';
```
- [ ] **Step 4: Extend entity and DTO**
```java
private String sceneRef;
private String videoPrompt;
```
```java
dto.setSceneRef(s.getSceneRef());
dto.setVideoPrompt(s.getVideoPrompt());
```
- [ ] **Step 5: Add mapper helpers for follow-up tasks**
```java
@Select("SELECT * FROM storyboards WHERE episode_id=#{episodeId} AND tenant_id=#{tenantId} AND (video_prompt IS NULL OR video_prompt='') ORDER BY sequence_num ASC")
List<Storyboard> findMissingVideoPromptByEpisode(Long episodeId, Long tenantId);
```
- [ ] **Step 6: Run test to verify it passes**
Run:
```bash
mvn -q -pl yaoai-comic-studio/yaoai-bootstrap -am -Dtest=StoryboardPipelineServiceImplTest#from_shouldExposeSceneRefAndVideoPrompt test
```
Expected: PASS
- [ ] **Step 7: Commit**
```bash
git add yaoai-comic-studio/yaoai-bootstrap/src/main/resources/db/migration/V17__storyboard_scene_ref_video_prompt.sql yaoai-comic-studio/yaoai-domain/src/main/java/com/yaoai/domain/entity/Storyboard.java yaoai-comic-studio/yaoai-domain/src/main/java/com/yaoai/domain/mapper/StoryboardMapper.java yaoai-comic-studio/yaoai-api/src/main/java/com/yaoai/api/dto/ai/StoryboardDTO.java yaoai-comic-studio/yaoai-bootstrap/src/test/java/com/yaoai/pipeline/service/impl/StoryboardPipelineServiceImplTest.java
git commit -m "feat(storyboard): add scene ref and video prompt storage"
```
---
### Task 2: Normalize storyboard generation and persist prompt output
**Files:**
- Modify: `yaoai-comic-studio/yaoai-pipeline/src/main/java/com/yaoai/pipeline/service/StoryboardPipelineService.java`
- Modify: `yaoai-comic-studio/yaoai-pipeline/src/main/java/com/yaoai/pipeline/service/impl/StoryboardPipelineServiceImpl.java`
- Modify: `yaoai-comic-studio/yaoai-api/src/main/java/com/yaoai/api/controller/StoryboardController.java`
- Test: `yaoai-comic-studio/yaoai-bootstrap/src/test/java/com/yaoai/pipeline/service/impl/StoryboardPipelineServiceImplTest.java`
- [ ] **Step 1: Write failing tests for scene extraction, prompt persistence, and video prompt invalidation**
```java
@Test
void generateStoryboards_shouldSplitSceneRefFromCharacters() {
String raw = """
[{
"sequence_num": 1,
"scene_number": "001",
"short_description": "两人在咖啡馆对视",
"detailed_description": "晚间咖啡馆内，人物对坐，气氛克制",
"characters": "@林夏,@程野,@咖啡馆",
"start_frame_prompt": "cinematic cafe interior, two people facing each other",
"motion_script": "缓慢推进"
}]
""";
when(llmService.chat(any())).thenReturn(raw);
List<Storyboard> storyboards = service.generateStoryboards(11L, 22L, 33L);
assertEquals("@咖啡馆", storyboards.get(0).getSceneRef());
assertEquals("@林夏,@程野", storyboards.get(0).getCharacters());
}
@Test
void generatePrompt_shouldPersistVideoPrompt() {
Storyboard sb = new Storyboard();
sb.setId(7L);
sb.setTenantId(33L);
sb.setProjectId(22L);
sb.setSceneRef("@咖啡馆");
when(storyboardMapper.selectById(7L)).thenReturn(sb);
when(llmService.chat(any())).thenReturn("0-3s：中景推进，咖啡馆内安静对视");
String prompt = service.generatePrompt(7L, 33L);
assertEquals("0-3s：中景推进，咖啡馆内安静对视", prompt);
verify(storyboardMapper).updateById(argThat(updated ->
"0-3s：中景推进，咖啡馆内安静对视".equals(updated.getVideoPrompt())
));
}
@Test
void update_shouldClearVideoPromptWhenStoryboardBodyChanges() {
Storyboard existing = new Storyboard();
existing.setId(9L);
existing.setTenantId(33L);
existing.setShortDescription("旧描述");
existing.setVideoPrompt("旧视频提示词");
when(storyboardMapper.selectById(9L)).thenReturn(existing, existing);
Storyboard patch = new Storyboard();
patch.setShortDescription("新描述");
service.update(9L, 33L, patch);
verify(storyboardMapper).updateById(argThat(updated -> updated.getVideoPrompt() == null));
}
```
- [ ] **Step 2: Run the focused backend tests**
Run:
```bash
mvn -q -pl yaoai-comic-studio/yaoai-bootstrap -am -Dtest=StoryboardPipelineServiceImplTest test
```
Expected: FAIL because scene splitting, prompt persistence, and invalidation are not implemented.
- [ ] **Step 3: Extend the service API**
```java
String generatePrompt(Long storyboardId, Long tenantId);
List<Storyboard> populateMissingVideoPromptsByEpisode(Long episodeId, Long tenantId);
```
- [ ] **Step 4: Normalize model output and add helper methods**
```java
private record StoryboardRefs(String characters, String sceneRef) {}
private StoryboardRefs normalizeStoryboardRefs(String rawRefs, Long projectId, Long tenantId) {
List<Scene> scenes = sceneMapper.findByProject(projectId, tenantId);
Set<String> sceneRefs = scenes.stream()
.map(Scene::getName)
.filter(Objects::nonNull)
.map(name -> "@" + name)
.collect(Collectors.toSet());
List<String> tokens = Arrays.stream(nullSafe(rawRefs).split(","))
.map(String::trim)
.filter(token -> !token.isBlank())
.toList();
String sceneRef = tokens.stream().filter(sceneRefs::contains).findFirst().orElse(null);
String characters = tokens.stream().filter(token -> !sceneRefs.contains(token)).collect(Collectors.joining(","));
return new StoryboardRefs(characters, sceneRef);
}
```
- [ ] **Step 5: Persist `video_prompt` inside `generatePrompt` and add episode batch fill**
```java
Storyboard update = new Storyboard();
update.setId(sb.getId());
update.setVideoPrompt(prompt);
storyboardMapper.updateById(update);
return prompt;
```
```java
@Override
public List<Storyboard> populateMissingVideoPromptsByEpisode(Long episodeId, Long tenantId) {
List<Storyboard> pending = storyboardMapper.findMissingVideoPromptByEpisode(episodeId, tenantId);
for (Storyboard storyboard : pending) {
generatePrompt(storyboard.getId(), tenantId);
}
return storyboardMapper.findByEpisode(episodeId, tenantId);
}
```
- [ ] **Step 6: Clear `video_prompt` whenever storyboard body fields change**
```java
if (bodyFieldsChanged(existing, patch)) {
patch.setVideoPrompt(null);
}
```
- [ ] **Step 7: Keep controller contract unchanged but ensure prompt endpoint returns persisted data**
```java
String prompt = storyboardService.generatePrompt(storyboardId, tenantId);
return ApiResponse.success(Map.of("prompt", prompt));
```
- [ ] **Step 8: Re-run backend tests**
Run:
```bash
mvn -q -pl yaoai-comic-studio/yaoai-bootstrap -am -Dtest=StoryboardPipelineServiceImplTest test
```
Expected: PASS
- [ ] **Step 9: Commit**
```bash
git add yaoai-comic-studio/yaoai-pipeline/src/main/java/com/yaoai/pipeline/service/StoryboardPipelineService.java yaoai-comic-studio/yaoai-pipeline/src/main/java/com/yaoai/pipeline/service/impl/StoryboardPipelineServiceImpl.java yaoai-comic-studio/yaoai-api/src/main/java/com/yaoai/api/controller/StoryboardController.java yaoai-comic-studio/yaoai-bootstrap/src/test/java/com/yaoai/pipeline/service/impl/StoryboardPipelineServiceImplTest.java
git commit -m "feat(storyboard): persist scene refs and video prompts"
```
---
### Task 3: Teach Agent `storyboard_gen` to backfill prompts
**Files:**
- Modify: `yaoai-comic-studio/yaoai-agent/src/main/java/com/yaoai/agent/service/impl/AgentRunServiceImpl.java`
- Modify: `yaoai-comic-studio/yaoai-pipeline/src/main/java/com/yaoai/pipeline/service/StoryboardPipelineService.java`
- Test: `yaoai-comic-studio/yaoai-bootstrap/src/test/java/com/yaoai/agent/service/impl/AgentRunServiceImplTest.java`
- [ ] **Step 1: Write failing Agent tests**
```java
@Test
void storyboardGen_shouldPopulateMissingVideoPromptsAfterGeneratingStoryboards() {
Episode ep = new Episode();
ep.setId(101L);
ep.setEpisodeNumber(1);
when(outlinePipeline.generateEpisodes(anyLong(), anyLong(), anyLong())).thenReturn(List.of(ep));
when(storyboardPipeline.generateStoryboards(101L, 22L, 33L)).thenReturn(List.of(new Storyboard(), new Storyboard()));
when(storyboardPipeline.populateMissingVideoPromptsByEpisode(101L, 33L)).thenReturn(List.of(new Storyboard(), new Storyboard()));
service.startRun(22L, 33L, "生成完整短剧");
verify(storyboardPipeline).populateMissingVideoPromptsByEpisode(101L, 33L);
}
```
- [ ] **Step 2: Run Agent test to verify it fails**
Run:
```bash
mvn -q -pl yaoai-comic-studio/yaoai-bootstrap -am -Dtest=AgentRunServiceImplTest test
```
Expected: FAIL because Agent only generates storyboards today.
- [ ] **Step 3: Update the `storyboard_gen` execution block**
```java
for (Episode ep : episodes) {
List<Storyboard> boards = storyboardPipeline.generateStoryboards(ep.getId(), projectId, tenantId);
total += boards.size();
sseManager.emitLog(runId, "info", "第" + ep.getEpisodeNumber() + "集分镜：" + boards.size() + " 个镜头");
List<Storyboard> enriched = storyboardPipeline.populateMissingVideoPromptsByEpisode(ep.getId(), tenantId);
long promptCount = enriched.stream().filter(sb -> sb.getVideoPrompt() != null && !sb.getVideoPrompt().isBlank()).count();
sseManager.emitLog(runId, "info", "第" + ep.getEpisodeNumber() + "集视频提示词：" + promptCount + " 条已生成");
}
```
- [ ] **Step 4: Add bounded prompt-fill execution**
```java
private static final int STORYBOARD_PROMPT_CONCURRENCY = 3;
```
Use a fixed-size executor or `CompletableFuture` batch inside `populateMissingVideoPromptsByEpisode`, but keep cross-episode execution serial.
- [ ] **Step 5: Mark the summary output clearly**
```java
return "共生成 " + total + " 个分镜，并补全视频提示词";
```
- [ ] **Step 6: Re-run Agent and backend tests**
Run:
```bash
mvn -q -pl yaoai-comic-studio/yaoai-bootstrap -am -Dtest=StoryboardPipelineServiceImplTest,AgentRunServiceImplTest test
```
Expected: PASS
- [ ] **Step 7: Commit**
```bash
git add yaoai-comic-studio/yaoai-agent/src/main/java/com/yaoai/agent/service/impl/AgentRunServiceImpl.java yaoai-comic-studio/yaoai-pipeline/src/main/java/com/yaoai/pipeline/service/StoryboardPipelineService.java yaoai-comic-studio/yaoai-bootstrap/src/test/java/com/yaoai/agent/service/impl/AgentRunServiceImplTest.java
git commit -m "feat(agent): enrich storyboard generation with prompt backfill"
```
---
### Task 4: Expose and display the richer storyboard fields
**Files:**
- Modify: `doc/html/src/lib/api/ai.ts`
- Modify: `doc/html/src/hooks/useAi.ts`
- Modify: `doc/html/src/app/pages/StoryboardWorkspace.tsx`
- [ ] **Step 1: Add the frontend type fields**
```ts
export interface Storyboard {
id: string;
episodeId: string;
sequenceNum: number;
sceneNumber: string;
shortDescription: string;
detailedDescription: string;
characters: string;
sceneRef?: string | null;
videoPrompt?: string | null;
dialogues: string;
cameraDirection: string;
compositionGuide: string;
durationSeconds: number;
startFramePrompt: string;
motionScript: string;
notes: string;
status: string;
createdAt: string;
updatedAt: string;
}
```
- [ ] **Step 2: Keep the existing React Query hooks and only adjust the consumed shape**
```ts
mutationFn: ({ id, patch }: { id: string; patch: Record<string, unknown> }) =>
aiApi.updateStoryboard(projectId, id, patch),
```
No new route or new hook is needed.
- [ ] **Step 3: Add display blocks to the storyboard workspace**
```tsx
<div className="space-y-3">
<Field label="出镜角色" value={expanded.characters || "未标注"} />
<Field label="分镜场景" value={expanded.sceneRef || "未标注"} />
<Field label="首帧提示词" value={expanded.startFramePrompt || "未生成"} />
<Field label="镜头运动" value={expanded.motionScript || "未生成"} />
<Field label="视频提示词" value={expanded.videoPrompt || "未生成"} multiline />
</div>
```
- [ ] **Step 4: Show old data safely**
```tsx
const sceneRefDisplay = expanded.sceneRef?.trim() || inferSceneRefFromCharacters(expanded.characters) || "未标注";
```
- [ ] **Step 5: Build the frontend**
Run:
```bash
cd doc/html
npm run build
```
Expected: PASS
- [ ] **Step 6: Commit**
```bash
git add doc/html/src/lib/api/ai.ts doc/html/src/hooks/useAi.ts doc/html/src/app/pages/StoryboardWorkspace.tsx
git commit -m "feat(frontend): show enriched storyboard fields"
```
---
### Task 5: Full verification and cleanup
**Files:**
- Modify: `docs/superpowers/specs/2026-04-28-agent-storyboard-enrichment-design.md` only if implementation deviates
- [ ] **Step 1: Run backend compile**
Run:
```bash
mvn -q -pl yaoai-comic-studio/yaoai-bootstrap -am -DskipTests compile
```
Expected: PASS
- [ ] **Step 2: Run focused tests**
Run:
```bash
mvn -q -pl yaoai-comic-studio/yaoai-bootstrap -am -Dtest=StoryboardPipelineServiceImplTest,AgentRunServiceImplTest test
```
Expected: PASS
- [ ] **Step 3: Run frontend build**
Run:
```bash
cd doc/html
npm run build
```
Expected: PASS
- [ ] **Step 4: Smoke-check the critical user flows**
```text
1. 输入或上传剧本生成大纲、分集、角色与场景，确认这些前置流程不变。
2. 运行 Agent 制作，确认 storyboard_gen 日志包含“分镜数量”和“提示词数量”。
3. 打开分镜页，确认同一条分镜能看到角色、场景、首帧提示词、镜头运动、视频提示词。
4. 手动修改分镜正文后刷新数据，确认 videoPrompt 变为空。
5. 手动调用“生成提示词”后，确认 videoPrompt 被重新写回。
```
- [ ] **Step 5: Final commit**
```bash
git add .
git commit -m "feat(agent): enrich storyboard output for scene and video prompt"
```
---
## Self-Review
- Spec coverage:
- Database fields: covered in Task 1
- Scene extraction and prompt persistence: covered in Task 2
- Agent enrichment and prompt backfill: covered in Task 3
- Frontend display: covered in Task 4
- Verification and user-facing regression checks: covered in Task 5
- Placeholder scan:
- No `TODO` / `TBD`
- Every task has explicit files, commands, and code snippets
- Type consistency:
- Uses `sceneRef` and `videoPrompt` consistently in DTO and frontend
- Keeps database names as `scene_ref` and `video_prompt`
yaoai-comic-studio/yaoai-agent/src/main/java/com/yaoai/agent/service/impl/AgentRunServiceImpl.java
View file @ 2ce18b88
...@@ -187,14 +187,23 @@ public class AgentRunServiceImpl implements AgentRunService { ...@@ -187,14 +187,23 @@ public class AgentRunServiceImpl implements AgentRunService {
executeStep(runId, "storyboard_gen", () -> { executeStep(runId, "storyboard_gen", () -> {
sseManager.emitLog(runId, "agent", "[执行层] 生成分镜..."); sseManager.emitLog(runId, "agent", "[执行层] 生成分镜...");
int total = 0; int total = 0;
int promptTotal = 0;
for (Episode ep : episodes) { for (Episode ep : episodes) {
List<Storyboard> boards = storyboardPipeline.generateStoryboards( List<Storyboard> boards = storyboardPipeline.generateStoryboards(
ep.getId(), projectId, tenantId); ep.getId(), projectId, tenantId);
total += boards.size(); total += boards.size();
List<Storyboard> enriched = storyboardPipeline.populateMissingVideoPromptsByEpisode(
ep.getId(), tenantId);
long promptCount = enriched.stream()
.filter(sb -> sb.getVideoPrompt() != null && !sb.getVideoPrompt().isBlank())
.count();
promptTotal += (int) promptCount;
sseManager.emitLog(runId, "info",
"第 " + ep.getEpisodeNumber() + " 集视频提示词：" + promptCount + " 条已生成");
sseManager.emitLog(runId, "info", sseManager.emitLog(runId, "info",
"第 " + ep.getEpisodeNumber() + " 集分镜：" + boards.size() + " 个镜头"); "第 " + ep.getEpisodeNumber() + " 集分镜：" + boards.size() + " 个镜头");
} }
return "共生成 " + total + " 个分镜"; return "共生成 " + total + " 个分镜，并补全 " + promptTotal + " 条视频提示词";
}); });
checkPaused(runId); checkPaused(runId);
......
yaoai-comic-studio/yaoai-bootstrap/src/test/java/com/yaoai/agent/service/impl/AgentRunServiceImplTest.java 0 → 100644
View file @ 2ce18b88
package com.yaoai.agent.service.impl;
import com.yaoai.agent.sse.AgentSseManager;
import com.yaoai.ai.core.service.LlmService;
import com.yaoai.domain.entity.Episode;
import com.yaoai.domain.entity.GraphRun;
import com.yaoai.domain.entity.GraphStep;
import com.yaoai.domain.entity.Outline;
import com.yaoai.domain.entity.Storyboard;
import com.yaoai.domain.mapper.EpisodeMapper;
import com.yaoai.domain.mapper.GraphRunMapper;
import com.yaoai.domain.mapper.GraphStepMapper;
import com.yaoai.pipeline.service.AssetGenPipelineService;
import com.yaoai.pipeline.service.OutlinePipelineService;
import com.yaoai.pipeline.service.StoryboardPipelineService;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.test.util.ReflectionTestUtils;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.atLeastOnce;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@ExtendWith(MockitoExtension.class)
class AgentRunServiceImplTest {
@Mock
private GraphRunMapper graphRunMapper;
@Mock
private GraphStepMapper graphStepMapper;
@Mock
private EpisodeMapper episodeMapper;
@Mock
private AgentSseManager sseManager;
@Mock
private LlmService llmService;
@Mock
private OutlinePipelineService outlinePipeline;
@Mock
private AssetGenPipelineService assetGenPipeline;
@Mock
private StoryboardPipelineService storyboardPipeline;
@Test
void storyboardGen_shouldPopulateMissingVideoPromptsAfterGeneratingStoryboards() {
AgentRunServiceImpl service = new AgentRunServiceImpl(
graphRunMapper,
graphStepMapper,
episodeMapper,
sseManager,
llmService,
outlinePipeline,
assetGenPipeline,
storyboardPipeline
);
Long runId = 88L;
Long projectId = 22L;
Long tenantId = 33L;
GraphRun runningRun = new GraphRun();
runningRun.setId(runId);
runningRun.setStatus("RUNNING");
when(graphRunMapper.selectById(runId)).thenReturn(runningRun);
when(graphRunMapper.updateById(any(GraphRun.class))).thenReturn(1);
List<GraphStep> steps = List.of(
step(runId, "decision", 0),
step(runId, "outline_gen", 1),
step(runId, "episode_gen", 2),
step(runId, "supervision_1", 3),
step(runId, "asset_gen", 4),
step(runId, "storyboard_gen", 5),
step(runId, "supervision_2", 6)
);
when(graphStepMapper.findByRunId(runId)).thenReturn(steps);
when(graphStepMapper.updateById(any(GraphStep.class))).thenReturn(1);
when(llmService.chat(any())).thenReturn("SCORE: A");
Outline outline = new Outline();
outline.setId(77L);
outline.setTitle("outline");
outline.setEpisodeCount(1);
when(outlinePipeline.generateOutline(projectId, tenantId, "goal")).thenReturn(outline);
Episode episode = new Episode();
episode.setId(101L);
episode.setEpisodeNumber(1);
when(outlinePipeline.generateEpisodes(77L, projectId, tenantId)).thenReturn(List.of(episode));
when(assetGenPipeline.extractCharacters(projectId, tenantId)).thenReturn(List.of());
when(assetGenPipeline.extractScenes(projectId, tenantId)).thenReturn(List.of());
when(storyboardPipeline.generateStoryboards(101L, projectId, tenantId))
.thenReturn(List.of(new Storyboard(), new Storyboard()));
when(storyboardPipeline.populateMissingVideoPromptsByEpisode(101L, tenantId))
.thenReturn(List.of(storyboardWithPrompt(), storyboardWithPrompt()));
ReflectionTestUtils.invokeMethod(
service,
"executeRun",
runId,
projectId,
tenantId,
1001L,
"goal"
);
verify(storyboardPipeline).populateMissingVideoPromptsByEpisode(101L, tenantId);
ArgumentCaptor<GraphStep> stepCaptor = ArgumentCaptor.forClass(GraphStep.class);
verify(graphStepMapper, atLeastOnce()).updateById(stepCaptor.capture());
boolean storyboardOutputMentionedPrompts = stepCaptor.getAllValues().stream()
.anyMatch(step -> "storyboard_gen".equals(step.getStepKey())
&& "DONE".equals(step.getStatus())
&& step.getOutput() != null
&& step.getOutput().contains("视频提示词"));
assertTrue(storyboardOutputMentionedPrompts);
}
private static GraphStep step(Long runId, String stepKey, int seq) {
GraphStep step = new GraphStep();
step.setRunId(runId);
step.setStepKey(stepKey);
step.setStatus("PENDING");
step.setSeq(seq);
return step;
}
private static Storyboard storyboardWithPrompt() {
Storyboard storyboard = new Storyboard();
storyboard.setVideoPrompt("prompt");
return storyboard;
}
}
yaoai-comic-studio/yaoai-bootstrap/src/test/java/com/yaoai/pipeline/service/impl/StoryboardPipelineServiceImplTest.java
View file @ 2ce18b88
...@@ -315,7 +315,7 @@ class StoryboardPipelineServiceImplTest { ...@@ -315,7 +315,7 @@ class StoryboardPipelineServiceImplTest {
verify(storyboardMapper, never()).selectById(3L); verify(storyboardMapper, never()).selectById(3L);
ArgumentCaptor<Storyboard> updateCaptor = ArgumentCaptor.forClass(Storyboard.class); ArgumentCaptor<Storyboard> updateCaptor = ArgumentCaptor.forClass(Storyboard.class);
verify(storyboardMapper, times(2)).updateById(updateCaptor.capture()); verify(storyboardMapper, times(2)).updateById(updateCaptor.capture());
assertEquals(List.of(1L, 2L), updateCaptor.getAllValues().stream().map(Storyboard::getId).toList()); assertTrue(updateCaptor.getAllValues().stream().map(Storyboard::getId).toList().containsAll(List.of(1L, 2L)));
} }
private static Scene scene(String name, String sceneType) { private static Scene scene(String name, String sceneType) {
......
yaoai-comic-studio/yaoai-pipeline/src/main/java/com/yaoai/pipeline/service/impl/StoryboardPipelineServiceImpl.java
View file @ 2ce18b88
...@@ -32,12 +32,18 @@ import java.util.List; ...@@ -32,12 +32,18 @@ import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Objects; import java.util.Objects;
import java.util.Set; import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@Slf4j @Slf4j
@Service @Service
@RequiredArgsConstructor @RequiredArgsConstructor
public class StoryboardPipelineServiceImpl implements StoryboardPipelineService { public class StoryboardPipelineServiceImpl implements StoryboardPipelineService {
private static final int STORYBOARD_PROMPT_CONCURRENCY = 3;
private static final String SYSTEM_PROMPT_BASE = """ private static final String SYSTEM_PROMPT_BASE = """
You are a professional storyboard artist. You are a professional storyboard artist.
...@@ -363,8 +369,37 @@ public class StoryboardPipelineServiceImpl implements StoryboardPipelineService ...@@ -363,8 +369,37 @@ public class StoryboardPipelineServiceImpl implements StoryboardPipelineService
@Override @Override
public List<Storyboard> populateMissingVideoPromptsByEpisode(Long episodeId, Long tenantId) { public List<Storyboard> populateMissingVideoPromptsByEpisode(Long episodeId, Long tenantId) {
List<Storyboard> pending = storyboardMapper.findMissingVideoPromptByEpisode(episodeId, tenantId); List<Storyboard> pending = storyboardMapper.findMissingVideoPromptByEpisode(episodeId, tenantId);
for (Storyboard storyboard : pending) { if (pending.isEmpty()) {
generatePrompt(storyboard.getId(), tenantId); return storyboardMapper.findByEpisode(episodeId, tenantId);
}
Long userId = UserContext.get();
int poolSize = Math.min(STORYBOARD_PROMPT_CONCURRENCY, pending.size());
ExecutorService executor = Executors.newFixedThreadPool(poolSize);
try {
List<CompletableFuture<Void>> futures = pending.stream()
.map(storyboard -> CompletableFuture.runAsync(() -> {
if (userId != null) {
UserContext.set(userId);
}
try {
generatePrompt(storyboard.getId(), tenantId);
} finally {
UserContext.clear();
}
}, executor))
.toList();
for (CompletableFuture<Void> future : futures) {
future.join();
}
} catch (CompletionException e) {
if (e.getCause() instanceof RuntimeException runtimeException) {
throw runtimeException;
}
throw e;
} finally {
executor.shutdown();
} }
return storyboardMapper.findByEpisode(episodeId, tenantId); return storyboardMapper.findByEpisode(episodeId, tenantId);
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment