| 6 |
| """ |
| 7 | import argparse,hashlib,importlib.util,json,os,pickle,shlex,shutil,subprocess,sys,time,zipfile |
| 8 | from pathlib import Path |
| 9 | import numpy as np |
| 10 | from PIL import Image |
| 11 |
| 12 | ROOT=Path(__file__).resolve().parent |
| 13 |
| 14 | def run(argv,cwd,env): |
| 15 | print('+ '+' '.join(str(x)for x in argv),flush=True) |
| 16 | subprocess.run([str(x)for x in argv],cwd=cwd,env=env,check=True) |
| 17 |
| 18 | def materialize(source,dest): |
| 19 | if source.resolve()==dest.resolve():return |
| 20 | dest.parent.mkdir(parents=True,exist_ok=True) |
| 21 | shutil.copyfile(source,dest) |
| 22 |
| 23 | def main(): |
| 24 | ap=argparse.ArgumentParser(description=__doc__) |
| 25 | ap.add_argument('--reference',required=True,help='Original PNG-based emoji.pack, not EPK3') |
| 26 | ap.add_argument('--textures',required=True,help='Original 64x64 PNG ZIP') |
| 27 | ap.add_argument('--output',default='emoji.pack') |
| 28 | ap.add_argument('--base-pack',help='Optional verified EPK3 v2 pack (3,608,212 bytes)') |
| 29 | ap.add_argument('--work-dir',help='Reusable build directory; contains large intermediate files') |
| 30 | ap.add_argument('--workers',type=int,default=8) |
| 31 | ap.add_argument('--all-experiments',action='store_true',help='Also reproduce the rejected mask/dictionary experiments') |
| 32 | args=ap.parse_args() |
| 33 | if args.workers<1:ap.error('--workers must be positive') |
| 34 | reference=Path(args.reference).resolve();textures=Path(args.textures).resolve();output=Path(args.output).resolve() |
| 35 | if reference==output:ap.error('Do not overwrite the original input') |
| 36 | if reference.read_bytes()[:4]==b'EPK3':ap.error('--reference must be the original PNG-based pack') |
| 37 | work=Path(args.work_dir).resolve()if args.work_dir else output.parent/'emoji_build' |
| 38 | if work==ROOT:ap.error('Use a separate --work-dir') |
| 39 | for d in ['upload','out/emoji-v2','work/v2','work/v3']:(work/d).mkdir(parents=True,exist_ok=True) |
| 40 | env=os.environ.copy();env.update(OPENBLAS_NUM_THREADS='1',OMP_NUM_THREADS='1',EMOJI_WORKERS=str(args.workers)) |
| 41 | started=time.monotonic() |
| 42 | for name in ['optimizer_core.py','optimizer_visual.py','verify_emoji_pack.py','quality_seams.py']: |
| 43 | materialize(ROOT/name,work/'out/emoji-v2'/name) |
| 44 | materialize(reference,work/'upload/emoji.pack');materialize(textures,work/'upload/emoji_textures-5.zip') |
| 45 | base=work/'out/emoji-v2/emoji.pack' |
| 46 | if args.base_pack:materialize(Path(args.base_pack).resolve(),base) |
| 47 | else: |
| 48 | run([sys.executable,ROOT/'base_builder.py','--reference',reference,'--textures',textures, |
| 49 | '--output',base,'--workers',args.workers,'--cache',work/'base-cache'],work,env) |
| 50 | expected='258a9d78bb1a097377bca9355531403a83b71303222fa81b0bee111baec4186e' |
| 51 | actual=hashlib.sha256(base.read_bytes()).hexdigest() |
| 52 | if actual!=expected: |
| 53 | raise ValueError('This measured research build expects the supplied v2 baseline. Check inputs and pinned Pillow/libwebp versions.') |
| 54 | # Prepare exactly the same records consumed by every independent experiment. |
| 55 | sys.path.insert(0,str(ROOT)) |
| 56 | import optimizer_core as core |
| 57 | from verify_emoji_pack import Pack |
| 58 | records,ne,lookup,_=core.read_reference(reference,textures) |
| 59 | with open(work/'work/records.pkl','wb')as f:pickle.dump(records,f,protocol=5) |
| 60 | with zipfile.ZipFile(textures)as z: |
| 61 | originals=np.array([np.array(Image.open(z.open(f"{r['id']//4096}_{r['id']%4096}.png")).convert('RGBA'))for r in records[:ne]]) |
| 62 | np.savez_compressed(work/'work/originals.npz',rgba=originals) |
| 63 | pack=Pack(base);selected=[] |
| 64 | for i,e in enumerate(pack.entries): |
| 65 | parents=tuple(p for p in e[4:6]if p!=65535) |
| 66 | selected.append(dict(i=i,codec=e[7],pred=e[8],parents=parents,data=pack.blob(i),colors=e[6],channels=e[9],flags=e[10],paletteLength=pack.palette_length(e),cost=e[3],approx=bool(e[10]&128),transforms=tuple((e[10]>>(1+3*j))&7 for j in range(len(parents))))) |
| 67 | with open(work/'work/v2/final_selected.pkl','wb')as f:pickle.dump(selected,f,protocol=5) |
| 68 | current=np.array([pack.emoji(i)for i in range(ne)]);pack.close() |
| 69 | np.savez_compressed(work/'work/v2/final_images.npz',rgba=current) |
| 70 | for source in (ROOT/'experiments').iterdir(): |
| 71 | if source.is_file():materialize(source,work/'work/v3'/source.name) |
| 72 | run(shlex.split(os.environ.get('CXX','c++'))+['-O3','-shared','-fPIC','work/v3/joint.cpp','-o','work/v3/joint.so'],work,env) |
| 73 | stages=['joint_bench.py','joint_extra.py','select_joint.py','assemble_pack.py', |
| 74 | 'flexible_bench.py','flexible_extra.py','refit_halves.py','select_final.py'] |
| 75 | if args.all_experiments:stages+=['mask_bench.py','mask_select.py','dict_bench.py'] |
| 76 | for stage in stages:run([sys.executable,'work/v3/'+stage],work,env) |
| 77 | output.parent.mkdir(parents=True,exist_ok=True) |
| 78 | materialize(work/'work/v3/final.pack',output) |
| 79 | run([sys.executable,ROOT/'verify_emoji_pack.py',output,'--reference',reference,'--textures',textures, |
| 80 | '--compare-pack',base,'--json',output.with_suffix('.verification.json')],work,env) |
| 81 | result=dict(pack_bytes=output.stat().st_size,v2_bytes=base.stat().st_size, |
| 82 | saving_from_v2_bytes=base.stat().st_size-output.stat().st_size, |
| 83 | sha256=hashlib.sha256(output.read_bytes()).hexdigest(),build_seconds=round(time.monotonic()-started,2)) |
| 84 | output.with_suffix('.stats.json').write_text(json.dumps(result,indent=2)+'\n') |
| 85 | print(json.dumps(result,indent=2),flush=True) |
| 86 |
| 87 | if __name__=='__main__':main() |
| 88 |