借助 NVIDIA NeMo 生成用于金融 AI 研究的合成数据

config_builder = DataDesignerConfigBuilder()

# Model config for local vLLM-served Nemotron
local_model = ModelConfig(
    alias="local-nemotron",
    model="nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8",
    provider="local-vllm",
    inference_parameters=InferenceParameters(
        max_parallel_requests=448,
        temperature=0.95,
        top_p=0.95,
    ),
)
config_builder.add_model_config(local_model)

# Category sampler with DYNAMIC weights (updated each iteration)
config_builder.add_column(
    SamplerColumnConfig(
        name="category",
        sampler_type=SamplerType.CATEGORY,
        params=CategorySamplerParams(
            values=CATEGORIES,        # 12 topics + "Other"
            weights=current_weights,   # Adjusted after each iteration
        ),
    )
)

# LLM column for headline generation with few-shot examples
config_builder.add_column(
    LLMTextColumnConfig(
        name="headline",
        model_alias="local-nemotron",
        system_prompt="You are a financial news headline writer.",
        prompt=f"""Generate a realistic financial news headline \
for the category: {{{{ category }}}}

{format_examples_for_prompt(examples_by_category)}

Generate a single headline for the "{{{{ category }}}}" category.
Headline:""",
    )
)

SYSTEM:
  You are a financial news headline writer. Generate realistic,
  professional financial news headlines. Output ONLY the headline
  text, nothing else.

USER:
  Generate a realistic financial news headline for the category:
  {{ category }}

  Examples by category:
  - Earnings: "Google Parent Alphabet Posts $1.84 EPS, $63.5B Revenue"
  - Earnings: "Myriad Genetics Beats Q4 Estimates with 15% Revenue Surge"
  - Earnings: "Codec Solutions Posts Q3 Adjusted EPS $0.31, Surpasses ..."
  - Price Targets: "J.P. Morgan Cuts Target for Thermo Fisher, Downgrades ..."
  - Price Targets: "Mitsubishi UFJ Secures Buy Rating on Mitsui & Co. ..."
  - Price Targets: "Credence Resources Upgrades Target to $12.50 ..."
  - Stock price movement: "Equities Rise Broadly, Dow Gains 180 Points ..."
  - ...
  (3 examples x 12 topics + Other = up to 39 few-shot examples)

  Generate a single headline for the "{{ category }}" category.
  The headline should be 10-25 words long, realistic, and match
  the style of professional financial news.

  Headline:

workflow = TextSemanticDeduplicationWorkflow(
    input_path=input_path,
    output_path=output_path,
    cache_path=cache_path,
    perform_removal=True,
    # Embedding parameters
    text_field="headline",
    model_identifier="sentence-transformers/all-MiniLM-L6-v2",
    embedding_max_seq_length=512,
    embedding_model_inference_batch_size=1024,
    # Deduplication parameters
    n_clusters=500,     # Higher = smaller clusters = faster pairwise
    eps=0.10,           # eps = 1 - similarity_threshold (0.90)
    id_field="id", 
    # Among duplicates, keep the more representative copy
    ranking_strategy=RankingStrategy(   
        metadata_cols=["cosine_dist_to_cent"],
        ascending=True,  #Keep the most representative (closest to centroid)
    ),
    pairwise_batch_size=4096,
    input_filetype="parquet",
    output_filetype="parquet",
)
workflow.run()

def is_semantically_distinct(
    candidate_embedding, used_embeddings, threshold=0.80
):
    """Reject candidates >= threshold similar to ANY used example."""
    if used_embeddings.size == 0:
        return True
    candidate_norm = candidate_embedding / (
        np.linalg.norm(candidate_embedding) + 1e-9
    )
    used_norms = used_embeddings / (
        np.linalg.norm(used_embeddings, axis=1, keepdims=True) + 1e-9
    )
    similarities = np.dot(used_norms, candidate_norm)
    return np.max(similarities) < threshold

def compute_adjusted_weights(current_dist, target_dist, floor=0.005):
    adjusted = []
    for category in CATEGORIES:
        target = target_dist.get(category, 0.01)
        current = current_dist.get(category, 0.0)
        if current > 0:
            ratio = target / current           # > 1 = under-represented
            ratio = max(0.2, min(5.0, ratio))  # Clamp to avoid extremes
            weight = target * ratio
        else:
            weight = target * 2.0              # Missing category: boost
        adjusted.append(max(floor, weight))
    total = sum(adjusted)
    return [w / total for w in adjusted]       # Normalize to 1.0