Skip to content

Train june #196

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
jordiclive wants to merge 15 commits into
base: master
Choose a base branch
from
Draft

Train june #196

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
2dde85c
add full evaluation into training loop. Other training changes for A1…
May 17, 2023
6c4b6a7
no message
May 17, 2023
98598ba
fix format
May 17, 2023
f9af120
fix evaluation and config for 40gb
May 17, 2023
1e181af
Update evaluation.py
jordiclive May 18, 2023
efb6414
specify kwargs for non training usage
May 18, 2023
b8e5692
update with changes
Jun 6, 2023
ee61ded
update configs
Jun 6, 2023
a4619dc
no message
Jun 6, 2023
09c4e7f
no message
Jun 6, 2023
bc892cc
no message
Jun 6, 2023
55c5e4e
no message
Jun 6, 2023
35d6935
no message
Jun 6, 2023
1f3a1f1
no message
Jun 7, 2023
4e89102
no message
Jun 7, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 11 additions & 11 deletions bsmetadata/deepspeed_configs/v2.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,19 +30,19 @@
}
},
"zero_optimization": {
"stage": 1,
"allgather_partitions": true,
"allgather_bucket_size": 500000000,
"overlap_comm": true,
"reduce_scatter": true,
"reduce_bucket_size": 500000000,
"contiguous_gradients": true,
"cpu_offload": true
},
"gradient_accumulation_steps": 16,
"stage": 2,
"allgather_partitions": true,
"allgather_bucket_size": 2e8,
"overlap_comm": true,
"reduce_scatter": true,
"reduce_bucket_size": 2e8,
"contiguous_gradients": true,
"cpu_offload": false
},
"gradient_accumulation_steps": 2,
"gradient_clipping": "auto",
"steps_per_print": 100,
"train_batch_size": 256,
"train_batch_size": 512,
"train_micro_batch_size_per_gpu": "auto",
"wall_clock_breakdown": false
}
236 changes: 138 additions & 98 deletions bsmetadata/evaluation.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ def mean_loss_fn(
metadata_mask: torch.Tensor = None,
save_data: bool = False,
idx: int = None,
tokenizer = None,
) -> torch.Tensor:
"""Calculates the perplexity for a given batch.

Expand All @@ -62,15 +63,15 @@ def mean_loss_fn(
b = outputs.logits.size(0)
lm_logits = outputs.logits

lm_logits[:, :, 50257] = float("-inf")
lm_logits[:, :, 50258] = float("-inf")

labels = batch["labels"]
attention_mask = batch["attention_mask"]

shift_logits = lm_logits[..., :-1, :].contiguous()
shift_labels = labels[..., 1:].contiguous()
if metadata_mask is not None:
for special_tok in tokenizer.additional_special_tokens_ids:
shift_logits[:, :, special_tok] = torch.finfo(lm_logits.dtype).min

metadata_mask = metadata_mask.bool()
nonmetadata_cumsum = torch.cumsum(~metadata_mask, dim=-1)
first_nonmetadata = nonmetadata_cumsum == 1
Expand Down Expand Up @@ -133,7 +134,7 @@ def mean_loss_fn(
shift_labels.view(-1),
reduction="none",
).view(b, -1)

loss = torch.nan_to_num(loss)
if save_data:
# Save the non-masked tokens & their loss
suffix = "_meta" if metadata_mask is not None else ""
Expand Down Expand Up @@ -180,6 +181,8 @@ def get_mean_loss(
batch: Dict[str, torch.Tensor],
save_data: bool = False,
idx: int = None,
model=None,
tokenizer = None,
) -> torch.Tensor:
"""Prepares the arguments for perplexity calculation and passes them to the perplexity function.

Expand All @@ -197,7 +200,7 @@ def get_mean_loss(
metadata_mask = batch.pop("metadata_mask", None)
outputs = model(**batch)
batch["labels"] = labels
nll = mean_loss_fn(batch, outputs, metadata_mask, save_data=save_data, idx=idx)
nll = mean_loss_fn(batch, outputs, metadata_mask, save_data=save_data, idx=idx,tokenizer=tokenizer)
return nll


Expand Down Expand Up @@ -264,75 +267,28 @@ def create_metadata_prompt(example: Dict[str, Any], cfg: MetadataConfig) -> str:
return cfg.metadata_sep.join(sorted_metadata) + cfg.metadata_prefix_sep if sorted_metadata else ""


if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument(
"--repo_id",
type=str,
default="bs-modeling-metadata/checkpoints_all_04_23",
help="Repository ID for the model to compute perplexity for",
)
parser.add_argument(
"--subfolder",
type=str,
default="checkpoint-2000step",
help="subfolder in the respository with the specific checkpoint to evaluate perplexity for",
)
parser.add_argument(
"--config_file_path",
type=str,
help="The path actual_config.yaml if available, otherwise repo_id/actual_config.yaml or git clone's v2.yaml",
)
parser.add_argument(
"--output_file", type=str, default="evaluation.txt", help="Path to the file the perplexity is written to"
)
parser.add_argument("--no_cuda", action="store_true", help="If set to true, all computations are performed on CPU")
parser.add_argument(
"--save_data",
action="store_true",
help="If set to true, save tokens & losses",
)
parser.add_argument(
"--test",
action="store_true",
help="If set to true, the script runs in test mode and only takes 10 examples per dataset",
)
parser.add_argument(
"--max_n_examples",
type=int,
default=1500,
help="how many examples per metadata type to evaluate",
)
parser.add_argument(
"--metadata_to_test",
type=str,
default="html,entity,entity_paragraph,website_desc,generation_datasource,timestamp,title,generation_length_sentence,generation_length_text,url,paragraph",
help="metadata types to test",
)
parser.add_argument(
"--untrained",
action="store_true",
help="If set to true, will load gpt2-xl",
)
parser.add_argument(
"--prompt",
action="store_true",
help="If set to true, the script evaluates metadata in prompt style",
)

args = parser.parse_args()
print(f"Parameters: {args}")

# Load config
if args.config_file_path:
config_file_path = args.config_file_path
else:
try:
config_file_path = hf_hub_download(
repo_id=args.repo_id, filename="actual_config.yaml", use_auth_token=True
)
except Exception:
config_file_path = "bsmetadata/hydra_configs/v2.yaml"
def evaluate_main(
metadata_to_test: str = "title,html,entity_paragraph,website_desc,generation_datasource,timestamp",
output_file: str = "evaluation.txt",
repo_id: str = None,
subfolder: str = None,
test: bool = False,
max_n_examples: int = 1500,
prompt: bool = False,
no_cuda: bool = True,
save_data: bool = False,
untrained: bool = False,
config_file_path: str = None,
model: str = None,
tokenizer: str = None,
accelerator=None,
) -> dict:
# if config_file_path is None:
# try:
# config_file_path = hf_hub_download(repo_id=repo_id, filename="actual_config.yaml", use_auth_token=True)
# except Exception:
# config_file_path = "bsmetadata/hydra_configs/v2.yaml"
config_file_path = "/fsx/home-jordiclive/metadata/bsmetadata/hydra_configs/v2.yaml" #need to add this path to PYTHONPATH
repo_args = OmegaConf.load(config_file_path)
data_config = repo_args.data_config

Expand All @@ -341,15 +297,17 @@ def create_metadata_prompt(example: Dict[str, Any], cfg: MetadataConfig) -> str:

# Load model
print("Loading model...")
if args.untrained:
model = AutoModelForCausalLM.from_pretrained("gpt2-xl")
else:
model = AutoModelForCausalLM.from_pretrained(args.repo_id, subfolder=args.subfolder, use_auth_token=True)
model.eval().cuda() if not args.no_cuda else model.eval()

# Load tokenizer
tokenizer = AutoTokenizer.from_pretrained(repo_args.model_name)
tokenizer.pad_token = tokenizer.eos_token
if model is None or tokenizer is None:
if untrained:
model = AutoModelForCausalLM.from_pretrained("gpt2-xl")
tokenizer = AutoTokenizer.from_pretrained(repo_args.model_name)
tokenizer.pad_token = tokenizer.eos_token
else:
model = AutoModelForCausalLM.from_pretrained(repo_id, subfolder=subfolder, use_auth_token=True)
tokenizer = AutoTokenizer.from_pretrained(
"bs-modeling-metadata/checkpoints_all_04_23", subfolder="tokenizer", use_auth_token=True
)
model.eval().cuda() if not no_cuda else model.eval()

# Config preprocess function
cfg = data_config.metadata_config
Expand All @@ -358,7 +316,7 @@ def create_metadata_prompt(example: Dict[str, Any], cfg: MetadataConfig) -> str:
cfg.metadata_list.append("entity")
cfg.metadata_list.append("paragraph")

if args.prompt:
if prompt:
cfg.metadata_sep = "; " # Instead of " | "
cfg.metadata_prefix_sep = "" # Instead of " |||"; there's already an implicit " "
DatasourceProcessor.process_global = datasource_process_global_for_prompt
Expand All @@ -381,8 +339,8 @@ def create_metadata_prompt(example: Dict[str, Any], cfg: MetadataConfig) -> str:
"bs-modeling-metadata/c4-en-html-with-validation_metadata_url",
"bs-modeling-metadata/c4-en-html-with-validation_metadata_paragraph",
]
dataset_paths = [path for path in dataset_paths if path.split("_metadata_")[1] in args.metadata_to_test.split(",")]

dataset_paths = [path for path in dataset_paths if path.split("_metadata_")[1] in metadata_to_test.split(",")]
results = {}
for path in dataset_paths:
n_examples = 0
total_normal_len = []
Expand All @@ -394,11 +352,11 @@ def create_metadata_prompt(example: Dict[str, Any], cfg: MetadataConfig) -> str:
# Load validation dataset from hugging face
metadata_type = path.split("_metadata_")[1]
print(f"Loading {metadata_type} data...")
split = "validation" if not args.test else "validation[:10]"
split = "validation" if not test else "validation[:10]"
validation_dataset = load_dataset(path, use_auth_token=True, split=split)

data = []
max_n_examples_ord = len(str(args.max_n_examples))
max_n_examples_ord = len(str(max_n_examples))
for idx, example in tqdm(enumerate(validation_dataset), desc=f"Calculating perplexity for {metadata_type}..."):
# for idx in [136,]:
example = validation_dataset[idx]
Expand All @@ -409,7 +367,7 @@ def create_metadata_prompt(example: Dict[str, Any], cfg: MetadataConfig) -> str:
except Exception as e:
# Write error to output file and continue with next dataset
print(e)
with open(args.output_file, "a", encoding="utf8") as f:
with open(output_file, "a", encoding="utf8") as f:
f.write(f"=== RESULT [{metadata_type}] ===\n")
f.write(f"{e}\n\n")
exit_flag = True
Expand Down Expand Up @@ -445,7 +403,10 @@ def create_metadata_prompt(example: Dict[str, Any], cfg: MetadataConfig) -> str:
normal_batch = default_data_collator([normal_example])
metadata_example["labels"] = metadata_example["input_ids"]
metadata_batch = default_data_collator([metadata_example])
if not args.no_cuda:
if accelerator is not None:
normal_batch = {k: v.to(accelerator.device) for k, v in normal_batch.items()}
metadata_batch = {k: v.to(accelerator.device) for k, v in metadata_batch.items()}
elif not no_cuda:
normal_batch = {k: v.cuda() for k, v in normal_batch.items()}
metadata_batch = {k: v.cuda() for k, v in metadata_batch.items()}
if n_examples == 1:
Expand All @@ -461,12 +422,14 @@ def create_metadata_prompt(example: Dict[str, Any], cfg: MetadataConfig) -> str:
# rich.print(tokenizer.decode(metadata_batch["input_ids"][0]))

# Calculate nll (natural-log loss)
normal_nll, normal_example_len = get_mean_loss(normal_batch, save_data=args.save_data, idx=idx) # [0]
normal_nll, normal_example_len = get_mean_loss(
normal_batch, save_data=save_data, idx=idx, model=model,tokenizer=tokenizer
) # [0]
# print("PPL")
# print(normal_ppl)
total_normal_nll.append(normal_nll) # * normal_example_len
metadata_nll, metadata_example_len = get_mean_loss(
metadata_batch, save_data=args.save_data, idx=idx
metadata_batch, save_data=save_data, idx=idx, model=model,tokenizer=tokenizer
) # [0]
# print(metadata_ppl)
total_metadata_nll.append(metadata_nll) # * metadata_example_len
Expand Down Expand Up @@ -521,7 +484,7 @@ def create_metadata_prompt(example: Dict[str, Any], cfg: MetadataConfig) -> str:

# sys.exit()

if n_examples > args.max_n_examples:
if n_examples > max_n_examples:
break

if exit_flag:
Expand Down Expand Up @@ -554,9 +517,86 @@ def ppl(examples_mean_loss, examples_len):
else:
final_metadata_ppl = final_normal_ppl = 0

# Write results to output file
with open(args.output_file, "a", encoding="utf8") as f:
f.write(f"=== RESULT [{metadata_type}] ===\n")
f.write("Perplexity (metadata): {:>6,.3f}\n".format(final_metadata_ppl))
f.write("Perplexity (normal): {:>6,.3f}\n\n".format(final_normal_ppl))
results[metadata_type] = {"final_normal_ppl": final_normal_ppl, "final_metadata_ppl": final_metadata_ppl}
torch.save(data, "eva.data")
return results


if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument(
"--repo_id",
type=str,
default="bs-modeling-metadata/checkpoints_all_04_23",
help="Repository ID for the model to compute perplexity for",
)
parser.add_argument(
"--subfolder",
type=str,
default="checkpoint-2000step",
help="subfolder in the respository with the specific checkpoint to evaluate perplexity for",
)
parser.add_argument(
"--config_file_path",
type=str,
help="The path actual_config.yaml if available, otherwise repo_id/actual_config.yaml or git clone's v2.yaml",
)
parser.add_argument(
"--output_file", type=str, default="evaluation.txt", help="Path to the file the perplexity is written to"
)
parser.add_argument("--no_cuda", action="store_true", help="If set to true, all computations are performed on CPU")
parser.add_argument(
"--save_data",
action="store_true",
help="If set to true, save tokens & losses",
)
parser.add_argument(
"--test",
action="store_true",
help="If set to true, the script runs in test mode and only takes 10 examples per dataset",
)
parser.add_argument(
"--max_n_examples",
type=int,
default=1500,
help="how many examples per metadata type to evaluate",
)
parser.add_argument(
"--metadata_to_test",
type=str,
default="html,entity,entity_paragraph,website_desc,generation_datasource,timestamp,title,generation_length_sentence,generation_length_text,url,paragraph",
help="metadata types to test",
)
parser.add_argument(
"--untrained",
action="store_true",
help="If set to true, will load gpt2-xl",
)
parser.add_argument(
"--prompt",
action="store_true",
help="If set to true, the script evaluates metadata in prompt style",
)

args = parser.parse_args()
print(f"Parameters: {args}")
results = evaluate_main(
repo_id=args.repo_id,
subfolder=args.subfolder,
config_file_path=args.config_file_path,
output_file=args.output_file,
save_data=args.save_data,
test=args.test,
max_n_examples=args.max_n_examples,
metadata_to_test=args.metadata_to_test,
untrained=args.untrained,
prompt=args.prompt,
no_cuda=args.no_cuda,
)
# Load config
# Write results to output file
with open(args.output_file, "a", encoding="utf8") as f:
for k, v in results.items():
f.write(f"=== RESULT [{k}] ===\n")
f.write("Perplexity (metadata): {:>6,.3f}\n".format(v["final_metadata_ppl"]))
f.write("Perplexity (normal): {:>6,.3f}\n\n".format(v["final_normal_ppl"]))
Loading