{
  "name": "moar-v3",
  "arch": "lurk-gpt",
  "params": 5310720,
  "dtype": "float32",
  "model": {
    "vocab_size": 8192,
    "ctx": 256,
    "d_model": 256,
    "n_layer": 4,
    "n_head": 4,
    "mlp_hidden": 704,
    "rope_theta": 10000.0,
    "norm_eps": 1e-05,
    "dropout": 0.0,
    "arch": "lurk-gpt"
  },
  "tokenizer": {
    "type": "byte-level-bpe",
    "file": "tokenizer.json",
    "vocab_size": 8192,
    "doc_token": "<|doc|>",
    "doc_token_id": 0,
    "pad_token": "<|pad|>",
    "pad_token_id": 1
  },
  "train": {
    "rung": "tiny",
    "steps": 4166,
    "tokens_seen": 34127872,
    "seed": 1337,
    "batch_tokens": 8192,
    "micro_batch": 8,
    "grad_accum": 4,
    "lr": 0.0015,
    "warmup": 200,
    "optimizer": "adamw(0.9,0.95) wd=0.1 (matmuls only)",
    "schedule": "warmup + cosine to 0.1x",
    "grad_clip": 1.0,
    "precision": "fp32",
    "device": "mps (Apple M1)",
    "train_seconds": 2122
  },
  "dataset": {
    "source": "lurk crawl only",
    "tokens": 17426665,
    "docs": 7323,
    "train_tokens": 17061088,
    "val_tokens": 365577,
    "max_page_id": 8609
  },
  "val_loss": 3.5045
}