Files
novelty-seeking/frontend/src/App.tsx
gbanyan bc281b8e0a feat: Add Deduplication Agent with embedding and LLM methods
Implement a new Deduplication Agent that identifies and groups similar
transformation descriptions. Supports two deduplication methods:
- Embedding: Fast vector similarity comparison using cosine similarity
- LLM: Accurate pairwise semantic comparison (slower but more precise)

Backend changes:
- Add deduplication router with /deduplicate endpoint
- Add embedding_service for vector-based similarity
- Add llm_deduplication_service for LLM-based comparison
- Improve expert_transformation error handling and progress reporting

Frontend changes:
- Add DeduplicationPanel with interactive group visualization
- Add useDeduplication hook for state management
- Integrate deduplication tab in main App
- Add threshold slider and method selector in sidebar

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2025-12-22 20:26:17 +08:00

350 lines
14 KiB
TypeScript

import { useState, useRef, useCallback, useEffect } from 'react';
import { ConfigProvider, Layout, theme, Typography, Space, Tabs, Slider, Radio } from 'antd';
import { ApartmentOutlined, ThunderboltOutlined, FilterOutlined } from '@ant-design/icons';
import { ThemeToggle } from './components/ThemeToggle';
import { InputPanel } from './components/InputPanel';
import { TransformationInputPanel } from './components/TransformationInputPanel';
import { MindmapPanel } from './components/MindmapPanel';
import { TransformationPanel } from './components/TransformationPanel';
import { DeduplicationPanel } from './components/DeduplicationPanel';
import { useAttribute } from './hooks/useAttribute';
import { getModels } from './services/api';
import type { MindmapDAGRef } from './components/MindmapDAG';
import type { TransformationDAGRef } from './components/TransformationDAG';
import type { CategoryMode, ExpertSource, ExpertTransformationDAGResult, DeduplicationMethod } from './types';
const { Header, Sider, Content } = Layout;
const { Title } = Typography;
interface VisualSettings {
nodeSpacing: number;
fontSize: number;
}
function App() {
const [isDark, setIsDark] = useState(true);
const [activeTab, setActiveTab] = useState<string>('attribute');
const { loading, progress, error, currentResult, history, analyze, loadFromHistory } = useAttribute();
const [visualSettings, setVisualSettings] = useState<VisualSettings>({
nodeSpacing: 32,
fontSize: 14,
});
const mindmapRef = useRef<MindmapDAGRef>(null);
const transformationRef = useRef<TransformationDAGRef>(null);
// Transformation Agent settings
const [transformModel, setTransformModel] = useState<string>('');
const [transformTemperature, setTransformTemperature] = useState<number>(0.95);
const [expertConfig, setExpertConfig] = useState<{
expert_count: number;
keywords_per_expert: number;
custom_experts?: string[];
}>({
expert_count: 3,
keywords_per_expert: 1,
custom_experts: undefined,
});
const [customExpertsInput, setCustomExpertsInput] = useState('');
const [expertSource, setExpertSource] = useState<ExpertSource>('llm');
const [expertLanguage, setExpertLanguage] = useState<'en' | 'zh'>('en');
const [shouldStartTransform, setShouldStartTransform] = useState(false);
const [transformLoading, setTransformLoading] = useState(false);
const [transformationResult, setTransformationResult] = useState<ExpertTransformationDAGResult | null>(null);
// Deduplication settings
const [deduplicationThreshold, setDeduplicationThreshold] = useState(0.85);
const [deduplicationMethod, setDeduplicationMethod] = useState<DeduplicationMethod>('embedding');
// Available models from API
const [availableModels, setAvailableModels] = useState<string[]>([]);
// Fetch models on mount
useEffect(() => {
async function fetchModels() {
try {
const response = await getModels();
setAvailableModels(response.models);
// Set default model for transformation if not set
if (response.models.length > 0 && !transformModel) {
const defaultModel = response.models.find((m) => m.includes('qwen3')) || response.models[0];
setTransformModel(defaultModel);
}
} catch (err) {
console.error('Failed to fetch models:', err);
}
}
fetchModels();
}, []);
const handleAnalyze = async (
query: string,
model?: string,
temperature?: number,
chainCount?: number,
categoryMode?: CategoryMode,
customCategories?: string[],
suggestedCategoryCount?: number
) => {
await analyze(query, model, temperature, chainCount, categoryMode, customCategories, suggestedCategoryCount);
};
const handleResetView = useCallback(() => {
mindmapRef.current?.resetView();
}, []);
const handleTransform = useCallback(() => {
setShouldStartTransform(true);
}, []);
return (
<ConfigProvider
theme={{
algorithm: isDark ? theme.darkAlgorithm : theme.defaultAlgorithm,
}}
>
<Layout style={{ minHeight: '100vh' }}>
<Header
style={{
display: 'flex',
alignItems: 'center',
justifyContent: 'space-between',
padding: '0 24px',
background: isDark
? 'linear-gradient(90deg, #141414 0%, #1f1f1f 50%, #141414 100%)'
: 'linear-gradient(90deg, #fff 0%, #fafafa 50%, #fff 100%)',
borderBottom: isDark ? '1px solid #303030' : '1px solid #f0f0f0',
boxShadow: isDark
? '0 2px 8px rgba(0, 0, 0, 0.3)'
: '0 2px 8px rgba(0, 0, 0, 0.06)',
}}
>
<Space align="center" size="middle">
<ApartmentOutlined
style={{
fontSize: 24,
color: isDark ? '#177ddc' : '#1890ff',
}}
/>
<Title
level={4}
style={{
margin: 0,
background: isDark
? 'linear-gradient(90deg, #177ddc 0%, #69c0ff 100%)'
: 'linear-gradient(90deg, #1890ff 0%, #40a9ff 100%)',
WebkitBackgroundClip: 'text',
WebkitTextFillColor: 'transparent',
backgroundClip: 'text',
}}
>
Novelty Seeking
</Title>
</Space>
<ThemeToggle isDark={isDark} onToggle={setIsDark} />
</Header>
<Layout>
<Content
style={{
padding: 16,
height: 'calc(100vh - 64px)',
overflow: 'hidden',
}}
>
<Tabs
activeKey={activeTab}
onChange={setActiveTab}
style={{ height: '100%' }}
tabBarStyle={{ marginBottom: 8 }}
items={[
{
key: 'attribute',
label: (
<span>
<ApartmentOutlined style={{ marginRight: 8 }} />
Attribute Agent
</span>
),
children: (
<div style={{ height: 'calc(100vh - 140px)' }}>
<MindmapPanel
ref={mindmapRef}
data={currentResult}
loading={loading}
error={error}
isDark={isDark}
visualSettings={visualSettings}
/>
</div>
),
},
{
key: 'transformation',
label: (
<span>
<ThunderboltOutlined style={{ marginRight: 8 }} />
Transformation Agent
</span>
),
children: (
<div style={{ height: 'calc(100vh - 140px)' }}>
<TransformationPanel
ref={transformationRef}
attributeData={currentResult}
isDark={isDark}
model={transformModel}
temperature={transformTemperature}
expertConfig={expertConfig}
expertSource={expertSource}
expertLanguage={expertLanguage}
shouldStartTransform={shouldStartTransform}
onTransformComplete={() => setShouldStartTransform(false)}
onLoadingChange={setTransformLoading}
onResultsChange={setTransformationResult}
/>
</div>
),
},
{
key: 'deduplication',
label: (
<span>
<FilterOutlined style={{ marginRight: 8 }} />
Deduplication
</span>
),
children: (
<div style={{ height: 'calc(100vh - 140px)' }}>
<DeduplicationPanel
transformationResult={transformationResult}
isDark={isDark}
threshold={deduplicationThreshold}
onThresholdChange={setDeduplicationThreshold}
method={deduplicationMethod}
onMethodChange={setDeduplicationMethod}
/>
</div>
),
},
]}
/>
</Content>
<Sider
width={350}
theme={isDark ? 'dark' : 'light'}
style={{
height: 'calc(100vh - 64px)',
overflow: 'auto',
}}
>
{activeTab === 'attribute' && (
<InputPanel
loading={loading}
progress={progress}
history={history}
currentResult={currentResult}
onAnalyze={handleAnalyze}
onLoadHistory={loadFromHistory}
onResetView={handleResetView}
visualSettings={visualSettings}
onVisualSettingsChange={setVisualSettings}
/>
)}
{activeTab === 'transformation' && (
<TransformationInputPanel
onTransform={handleTransform}
loading={transformLoading}
hasData={!!currentResult}
isDark={isDark}
model={transformModel}
temperature={transformTemperature}
expertConfig={expertConfig}
customExpertsInput={customExpertsInput}
expertSource={expertSource}
expertLanguage={expertLanguage}
onModelChange={setTransformModel}
onTemperatureChange={setTransformTemperature}
onExpertConfigChange={setExpertConfig}
onCustomExpertsInputChange={setCustomExpertsInput}
onExpertSourceChange={setExpertSource}
onExpertLanguageChange={setExpertLanguage}
availableModels={availableModels}
/>
)}
{activeTab === 'deduplication' && (
<div style={{ padding: 16 }}>
<Typography.Title level={5} style={{ marginBottom: 16 }}>
<FilterOutlined style={{ marginRight: 8 }} />
Deduplication Settings
</Typography.Title>
{/* Method Selection */}
<div style={{ marginBottom: 20 }}>
<Typography.Text strong style={{ display: 'block', marginBottom: 8 }}>
Method
</Typography.Text>
<Radio.Group
value={deduplicationMethod}
onChange={(e) => setDeduplicationMethod(e.target.value)}
buttonStyle="solid"
style={{ width: '100%' }}
>
<Radio.Button value="embedding" style={{ width: '50%', textAlign: 'center' }}>
Embedding
</Radio.Button>
<Radio.Button value="llm" style={{ width: '50%', textAlign: 'center' }}>
LLM Judge
</Radio.Button>
</Radio.Group>
<Typography.Text type="secondary" style={{ display: 'block', marginTop: 8, fontSize: 12 }}>
{deduplicationMethod === 'embedding'
? 'Fast vector similarity comparison'
: 'Accurate but slower pairwise LLM comparison'}
</Typography.Text>
</div>
{/* Threshold Slider - Only for Embedding method */}
{deduplicationMethod === 'embedding' && (
<div style={{ marginBottom: 20 }}>
<Typography.Text strong style={{ display: 'block', marginBottom: 8 }}>
Similarity Threshold
</Typography.Text>
<Typography.Text type="secondary" style={{ display: 'block', marginBottom: 12, fontSize: 12 }}>
Higher = stricter matching, fewer groups
</Typography.Text>
<Slider
min={0.5}
max={1.0}
step={0.05}
value={deduplicationThreshold}
onChange={setDeduplicationThreshold}
marks={{
0.5: '50%',
0.7: '70%',
0.85: '85%',
1.0: '100%',
}}
tooltip={{ formatter: (val) => `${((val ?? 0) * 100).toFixed(0)}%` }}
/>
<Typography.Text type="secondary" style={{ fontSize: 12 }}>
Current: {(deduplicationThreshold * 100).toFixed(0)}% similarity required
</Typography.Text>
</div>
)}
{/* LLM Warning */}
{deduplicationMethod === 'llm' && (
<Typography.Text type="warning" style={{ display: 'block', fontSize: 12 }}>
Note: LLM method requires N*(N-1)/2 comparisons. May take longer for many descriptions.
</Typography.Text>
)}
</div>
)}
</Sider>
</Layout>
</Layout>
</ConfigProvider>
);
}
export default App;