Implement a new Deduplication Agent that identifies and groups similar transformation descriptions. Supports two deduplication methods: - Embedding: Fast vector similarity comparison using cosine similarity - LLM: Accurate pairwise semantic comparison (slower but more precise) Backend changes: - Add deduplication router with /deduplicate endpoint - Add embedding_service for vector-based similarity - Add llm_deduplication_service for LLM-based comparison - Improve expert_transformation error handling and progress reporting Frontend changes: - Add DeduplicationPanel with interactive group visualization - Add useDeduplication hook for state management - Integrate deduplication tab in main App - Add threshold slider and method selector in sidebar 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
350 lines
14 KiB
TypeScript
350 lines
14 KiB
TypeScript
import { useState, useRef, useCallback, useEffect } from 'react';
|
|
import { ConfigProvider, Layout, theme, Typography, Space, Tabs, Slider, Radio } from 'antd';
|
|
import { ApartmentOutlined, ThunderboltOutlined, FilterOutlined } from '@ant-design/icons';
|
|
import { ThemeToggle } from './components/ThemeToggle';
|
|
import { InputPanel } from './components/InputPanel';
|
|
import { TransformationInputPanel } from './components/TransformationInputPanel';
|
|
import { MindmapPanel } from './components/MindmapPanel';
|
|
import { TransformationPanel } from './components/TransformationPanel';
|
|
import { DeduplicationPanel } from './components/DeduplicationPanel';
|
|
import { useAttribute } from './hooks/useAttribute';
|
|
import { getModels } from './services/api';
|
|
import type { MindmapDAGRef } from './components/MindmapDAG';
|
|
import type { TransformationDAGRef } from './components/TransformationDAG';
|
|
import type { CategoryMode, ExpertSource, ExpertTransformationDAGResult, DeduplicationMethod } from './types';
|
|
|
|
const { Header, Sider, Content } = Layout;
|
|
const { Title } = Typography;
|
|
|
|
interface VisualSettings {
|
|
nodeSpacing: number;
|
|
fontSize: number;
|
|
}
|
|
|
|
function App() {
|
|
const [isDark, setIsDark] = useState(true);
|
|
const [activeTab, setActiveTab] = useState<string>('attribute');
|
|
const { loading, progress, error, currentResult, history, analyze, loadFromHistory } = useAttribute();
|
|
const [visualSettings, setVisualSettings] = useState<VisualSettings>({
|
|
nodeSpacing: 32,
|
|
fontSize: 14,
|
|
});
|
|
const mindmapRef = useRef<MindmapDAGRef>(null);
|
|
const transformationRef = useRef<TransformationDAGRef>(null);
|
|
|
|
// Transformation Agent settings
|
|
const [transformModel, setTransformModel] = useState<string>('');
|
|
const [transformTemperature, setTransformTemperature] = useState<number>(0.95);
|
|
const [expertConfig, setExpertConfig] = useState<{
|
|
expert_count: number;
|
|
keywords_per_expert: number;
|
|
custom_experts?: string[];
|
|
}>({
|
|
expert_count: 3,
|
|
keywords_per_expert: 1,
|
|
custom_experts: undefined,
|
|
});
|
|
const [customExpertsInput, setCustomExpertsInput] = useState('');
|
|
const [expertSource, setExpertSource] = useState<ExpertSource>('llm');
|
|
const [expertLanguage, setExpertLanguage] = useState<'en' | 'zh'>('en');
|
|
const [shouldStartTransform, setShouldStartTransform] = useState(false);
|
|
const [transformLoading, setTransformLoading] = useState(false);
|
|
const [transformationResult, setTransformationResult] = useState<ExpertTransformationDAGResult | null>(null);
|
|
|
|
// Deduplication settings
|
|
const [deduplicationThreshold, setDeduplicationThreshold] = useState(0.85);
|
|
const [deduplicationMethod, setDeduplicationMethod] = useState<DeduplicationMethod>('embedding');
|
|
|
|
// Available models from API
|
|
const [availableModels, setAvailableModels] = useState<string[]>([]);
|
|
|
|
// Fetch models on mount
|
|
useEffect(() => {
|
|
async function fetchModels() {
|
|
try {
|
|
const response = await getModels();
|
|
setAvailableModels(response.models);
|
|
// Set default model for transformation if not set
|
|
if (response.models.length > 0 && !transformModel) {
|
|
const defaultModel = response.models.find((m) => m.includes('qwen3')) || response.models[0];
|
|
setTransformModel(defaultModel);
|
|
}
|
|
} catch (err) {
|
|
console.error('Failed to fetch models:', err);
|
|
}
|
|
}
|
|
fetchModels();
|
|
}, []);
|
|
|
|
const handleAnalyze = async (
|
|
query: string,
|
|
model?: string,
|
|
temperature?: number,
|
|
chainCount?: number,
|
|
categoryMode?: CategoryMode,
|
|
customCategories?: string[],
|
|
suggestedCategoryCount?: number
|
|
) => {
|
|
await analyze(query, model, temperature, chainCount, categoryMode, customCategories, suggestedCategoryCount);
|
|
};
|
|
|
|
const handleResetView = useCallback(() => {
|
|
mindmapRef.current?.resetView();
|
|
}, []);
|
|
|
|
const handleTransform = useCallback(() => {
|
|
setShouldStartTransform(true);
|
|
}, []);
|
|
|
|
return (
|
|
<ConfigProvider
|
|
theme={{
|
|
algorithm: isDark ? theme.darkAlgorithm : theme.defaultAlgorithm,
|
|
}}
|
|
>
|
|
<Layout style={{ minHeight: '100vh' }}>
|
|
<Header
|
|
style={{
|
|
display: 'flex',
|
|
alignItems: 'center',
|
|
justifyContent: 'space-between',
|
|
padding: '0 24px',
|
|
background: isDark
|
|
? 'linear-gradient(90deg, #141414 0%, #1f1f1f 50%, #141414 100%)'
|
|
: 'linear-gradient(90deg, #fff 0%, #fafafa 50%, #fff 100%)',
|
|
borderBottom: isDark ? '1px solid #303030' : '1px solid #f0f0f0',
|
|
boxShadow: isDark
|
|
? '0 2px 8px rgba(0, 0, 0, 0.3)'
|
|
: '0 2px 8px rgba(0, 0, 0, 0.06)',
|
|
}}
|
|
>
|
|
<Space align="center" size="middle">
|
|
<ApartmentOutlined
|
|
style={{
|
|
fontSize: 24,
|
|
color: isDark ? '#177ddc' : '#1890ff',
|
|
}}
|
|
/>
|
|
<Title
|
|
level={4}
|
|
style={{
|
|
margin: 0,
|
|
background: isDark
|
|
? 'linear-gradient(90deg, #177ddc 0%, #69c0ff 100%)'
|
|
: 'linear-gradient(90deg, #1890ff 0%, #40a9ff 100%)',
|
|
WebkitBackgroundClip: 'text',
|
|
WebkitTextFillColor: 'transparent',
|
|
backgroundClip: 'text',
|
|
}}
|
|
>
|
|
Novelty Seeking
|
|
</Title>
|
|
</Space>
|
|
<ThemeToggle isDark={isDark} onToggle={setIsDark} />
|
|
</Header>
|
|
<Layout>
|
|
<Content
|
|
style={{
|
|
padding: 16,
|
|
height: 'calc(100vh - 64px)',
|
|
overflow: 'hidden',
|
|
}}
|
|
>
|
|
<Tabs
|
|
activeKey={activeTab}
|
|
onChange={setActiveTab}
|
|
style={{ height: '100%' }}
|
|
tabBarStyle={{ marginBottom: 8 }}
|
|
items={[
|
|
{
|
|
key: 'attribute',
|
|
label: (
|
|
<span>
|
|
<ApartmentOutlined style={{ marginRight: 8 }} />
|
|
Attribute Agent
|
|
</span>
|
|
),
|
|
children: (
|
|
<div style={{ height: 'calc(100vh - 140px)' }}>
|
|
<MindmapPanel
|
|
ref={mindmapRef}
|
|
data={currentResult}
|
|
loading={loading}
|
|
error={error}
|
|
isDark={isDark}
|
|
visualSettings={visualSettings}
|
|
/>
|
|
</div>
|
|
),
|
|
},
|
|
{
|
|
key: 'transformation',
|
|
label: (
|
|
<span>
|
|
<ThunderboltOutlined style={{ marginRight: 8 }} />
|
|
Transformation Agent
|
|
</span>
|
|
),
|
|
children: (
|
|
<div style={{ height: 'calc(100vh - 140px)' }}>
|
|
<TransformationPanel
|
|
ref={transformationRef}
|
|
attributeData={currentResult}
|
|
isDark={isDark}
|
|
model={transformModel}
|
|
temperature={transformTemperature}
|
|
expertConfig={expertConfig}
|
|
expertSource={expertSource}
|
|
expertLanguage={expertLanguage}
|
|
shouldStartTransform={shouldStartTransform}
|
|
onTransformComplete={() => setShouldStartTransform(false)}
|
|
onLoadingChange={setTransformLoading}
|
|
onResultsChange={setTransformationResult}
|
|
/>
|
|
</div>
|
|
),
|
|
},
|
|
{
|
|
key: 'deduplication',
|
|
label: (
|
|
<span>
|
|
<FilterOutlined style={{ marginRight: 8 }} />
|
|
Deduplication
|
|
</span>
|
|
),
|
|
children: (
|
|
<div style={{ height: 'calc(100vh - 140px)' }}>
|
|
<DeduplicationPanel
|
|
transformationResult={transformationResult}
|
|
isDark={isDark}
|
|
threshold={deduplicationThreshold}
|
|
onThresholdChange={setDeduplicationThreshold}
|
|
method={deduplicationMethod}
|
|
onMethodChange={setDeduplicationMethod}
|
|
/>
|
|
</div>
|
|
),
|
|
},
|
|
]}
|
|
/>
|
|
</Content>
|
|
<Sider
|
|
width={350}
|
|
theme={isDark ? 'dark' : 'light'}
|
|
style={{
|
|
height: 'calc(100vh - 64px)',
|
|
overflow: 'auto',
|
|
}}
|
|
>
|
|
{activeTab === 'attribute' && (
|
|
<InputPanel
|
|
loading={loading}
|
|
progress={progress}
|
|
history={history}
|
|
currentResult={currentResult}
|
|
onAnalyze={handleAnalyze}
|
|
onLoadHistory={loadFromHistory}
|
|
onResetView={handleResetView}
|
|
visualSettings={visualSettings}
|
|
onVisualSettingsChange={setVisualSettings}
|
|
/>
|
|
)}
|
|
{activeTab === 'transformation' && (
|
|
<TransformationInputPanel
|
|
onTransform={handleTransform}
|
|
loading={transformLoading}
|
|
hasData={!!currentResult}
|
|
isDark={isDark}
|
|
model={transformModel}
|
|
temperature={transformTemperature}
|
|
expertConfig={expertConfig}
|
|
customExpertsInput={customExpertsInput}
|
|
expertSource={expertSource}
|
|
expertLanguage={expertLanguage}
|
|
onModelChange={setTransformModel}
|
|
onTemperatureChange={setTransformTemperature}
|
|
onExpertConfigChange={setExpertConfig}
|
|
onCustomExpertsInputChange={setCustomExpertsInput}
|
|
onExpertSourceChange={setExpertSource}
|
|
onExpertLanguageChange={setExpertLanguage}
|
|
availableModels={availableModels}
|
|
/>
|
|
)}
|
|
{activeTab === 'deduplication' && (
|
|
<div style={{ padding: 16 }}>
|
|
<Typography.Title level={5} style={{ marginBottom: 16 }}>
|
|
<FilterOutlined style={{ marginRight: 8 }} />
|
|
Deduplication Settings
|
|
</Typography.Title>
|
|
|
|
{/* Method Selection */}
|
|
<div style={{ marginBottom: 20 }}>
|
|
<Typography.Text strong style={{ display: 'block', marginBottom: 8 }}>
|
|
Method
|
|
</Typography.Text>
|
|
<Radio.Group
|
|
value={deduplicationMethod}
|
|
onChange={(e) => setDeduplicationMethod(e.target.value)}
|
|
buttonStyle="solid"
|
|
style={{ width: '100%' }}
|
|
>
|
|
<Radio.Button value="embedding" style={{ width: '50%', textAlign: 'center' }}>
|
|
Embedding
|
|
</Radio.Button>
|
|
<Radio.Button value="llm" style={{ width: '50%', textAlign: 'center' }}>
|
|
LLM Judge
|
|
</Radio.Button>
|
|
</Radio.Group>
|
|
<Typography.Text type="secondary" style={{ display: 'block', marginTop: 8, fontSize: 12 }}>
|
|
{deduplicationMethod === 'embedding'
|
|
? 'Fast vector similarity comparison'
|
|
: 'Accurate but slower pairwise LLM comparison'}
|
|
</Typography.Text>
|
|
</div>
|
|
|
|
{/* Threshold Slider - Only for Embedding method */}
|
|
{deduplicationMethod === 'embedding' && (
|
|
<div style={{ marginBottom: 20 }}>
|
|
<Typography.Text strong style={{ display: 'block', marginBottom: 8 }}>
|
|
Similarity Threshold
|
|
</Typography.Text>
|
|
<Typography.Text type="secondary" style={{ display: 'block', marginBottom: 12, fontSize: 12 }}>
|
|
Higher = stricter matching, fewer groups
|
|
</Typography.Text>
|
|
<Slider
|
|
min={0.5}
|
|
max={1.0}
|
|
step={0.05}
|
|
value={deduplicationThreshold}
|
|
onChange={setDeduplicationThreshold}
|
|
marks={{
|
|
0.5: '50%',
|
|
0.7: '70%',
|
|
0.85: '85%',
|
|
1.0: '100%',
|
|
}}
|
|
tooltip={{ formatter: (val) => `${((val ?? 0) * 100).toFixed(0)}%` }}
|
|
/>
|
|
<Typography.Text type="secondary" style={{ fontSize: 12 }}>
|
|
Current: {(deduplicationThreshold * 100).toFixed(0)}% similarity required
|
|
</Typography.Text>
|
|
</div>
|
|
)}
|
|
|
|
{/* LLM Warning */}
|
|
{deduplicationMethod === 'llm' && (
|
|
<Typography.Text type="warning" style={{ display: 'block', fontSize: 12 }}>
|
|
Note: LLM method requires N*(N-1)/2 comparisons. May take longer for many descriptions.
|
|
</Typography.Text>
|
|
)}
|
|
</div>
|
|
)}
|
|
</Sider>
|
|
</Layout>
|
|
</Layout>
|
|
</ConfigProvider>
|
|
);
|
|
}
|
|
|
|
export default App;
|