| 6 |
| Use --lossless to disable fitted palettes and lossy WebP candidates. |
| 7 | """ |
| 8 | import argparse, hashlib, io, json, pickle, struct, time, zipfile, zlib |
| 9 | from pathlib import Path |
| 10 | from collections import Counter |
| 11 | import numpy as np |
| 12 | from scipy.ndimage import uniform_filter |
| 13 | from PIL import Image, features |
| 14 | import optimizer_core as old |
| 15 | import optimizer_visual as visual |
| 16 |
| 17 | def custom(i,codec,pred,parents,ts,data,pal=None,approx=False): |
| 18 | if pal is None:o=old.option(i,codec,pred,parents,data) |
| 19 | else: |
| 20 | flags=0;pb=pal.astype(np.uint8).tobytes() |
| 21 | if all(v==((v>>1)*2+(v>>7)) for v in pb):pb=old.pack7(pb);flags=1 |
| 22 | o=dict(i=i,codec=codec,pred=pred,parents=parents,data=pb+data,colors=len(pal),channels=3,flags=flags,paletteLength=len(pb),cost=len(pb)+len(data)) |
| 23 | o['flags'] |= ((ts[0] if ts else 0)<<1)|((ts[1] if len(ts)>1 else 0)<<4)|(128 if approx else 0) |
| 24 | o['transforms']=ts;o['approx']=approx |
| 25 | return o |
| 26 | def pixels(o): |
| 27 | i=o['i'];b=o['data'];ch=o['channels'];n=o['colors'];plen=o['paletteLength'] |
| 28 | if o['codec']==3:a=np.array(Image.open(io.BytesIO(b)).convert('RGBA')) |
| 29 | else: |
| 30 | if o['flags']&1: |
| 31 | vals=[] |
| 32 | for j in range(n*ch): |
| 33 | bit=j*7;p=bit>>3;s=bit&7;v=b[p]>>s |
| 34 | if s>1:v|=b[p+1]<<(8-s) |
| 35 | v&=127;vals.append(v*2+(v>>6)) |
| 36 | pal=np.array(vals,np.uint8).reshape(n,ch) |
| 37 | else:pal=np.frombuffer(b[:plen],np.uint8).reshape(n,ch) |
| 38 | maps=[visual.trans(R[p]['idx'],t) for p,t in zip(o['parents'],o['transforms'])] |
| 39 | if o['codec']==4:idx=maps[0] |
| 40 | else: |
| 41 | lens=[len(R[p]['pal']) for p in o['parents']];raw=zlib.decompress(b[plen:],-15);lut=np.frombuffer(raw[:sum(lens)],np.uint8) |
| 42 | lm=lut[:lens[0]][maps[0]][:,:32];rm=lut[lens[0]:][maps[1]][:,32:];idx=np.concatenate([lm,rm],axis=1) |
| 43 | a=np.empty((64,64,4),np.uint8);a[:,:,:3]=pal[idx];a[:,:,3]=255 |
| 44 | if R[i]['mask']!=65535:a[:,:,3]=BASE[i,:,:,3] |
| 45 | return a |
| 46 |
| 47 |
| 48 | def assemble(base_half, mirrors, templates, half_templates, native, lossless=False): |
| 49 | roots, ind, edges = old.prepare() |
| 50 | for i, options in enumerate(base_half): |
| 51 | for _, a, b, codec, data in options: |
| 52 | edges[i].append(custom(i, codec, 3, (a, b), (0, 0), data[2:-4] if codec == 0 else data)) |
| 53 | if i < NE and R[i]['mask'] != 65535: |
| 54 | ln = len(R[a]['pal']) + len(R[b]['pal']) |
| 55 | if codec == 0: |
| 56 | raw = visual.zlib.decompress(data); lut = raw[:ln] |
| 57 | delta = np.frombuffer(raw[ln:], np.uint8).reshape(64, 64).copy() |
| 58 | else: |
| 59 | lut = data[:ln] |
| 60 | delta = np.array(Image.open(io.BytesIO(data[ln:])).convert('L')) |
| 61 | delta[BASE[i, :, :, 3] == 0] = 0 |
| 62 | zz = visual.zlib.compress(lut + delta.tobytes(), 9)[2:-4] |
| 63 | edges[i].append(custom(i, 0, 3, (a, b), (0, 0), zz)) |
| 64 | for i, options in enumerate(mirrors): |
| 65 | for o in options: |
| 66 | j, t, pred = o['parent'], o['transform'], o['pred'] |
| 67 | edges[i].append(custom(i, 0, pred, (j,), (t,), o['zlib'][2:-4])) |
| 68 | edges[i].append(custom(i, 1, pred, (j,), (t,), o['lut'] + o['webp'])) |
| 69 | if not lossless: |
| 70 | for i, (options, _) in enumerate(templates): |
| 71 | for _, j, t, pal, quality in options: |
| 72 | o = custom(i, 4, 4, (j,), (t,), b'', pal, True) |
| 73 | o['quality'] = quality; edges[i].append(o) |
| 74 | for i, options in enumerate(half_templates): |
| 75 | if not options: continue |
| 76 | ref = visual.metrics(BASE[i], O[i]) |
| 77 | for _, a, b, ta, tb, pal, zz, lm, rm in options: |
| 78 | o = custom(i, 0, 3, (a, b), (ta, tb), zz[2:-4], pal, True) |
| 79 | if visual.passes_global(pixels(o), O[i], ref): edges[i].append(o) |
| 80 | for i, (choice, ref, attempts) in enumerate(native): |
| 81 | options = [o for o in edges[i] if o.get('approx')] |
| 82 | if choice: |
| 83 | q, b, quality = choice |
| 84 | o = old.option(i, 3, 0, (), b) |
| 85 | o.update(approx=True, flags=128, native_webp_quality=q, quality=quality) |
| 86 | options.append(o) |
| 87 | if not options: continue |
| 88 | e0 = uniform_filter(visual.error(BASE[i], O[i]), size=5) |
| 89 | s0 = visual.structure(BASE[i], O[i]) |
| 90 | kept = [o for o in edges[i] if not o.get('approx')] |
| 91 | for o in options: |
| 92 | if visual.passes_local(pixels(o), O[i], e0, s0): |
| 93 | if 'native_webp_quality' in o: |
| 94 | if o['cost'] < ind[i]['cost']: ind[i] = o |
| 95 | else: kept.append(o) |
| 96 | edges[i] = kept |
| 97 | return old.select(roots, ind, edges, False) |
| 98 |
| 99 |
| 100 | def main(): |
| 101 | global R, O, NE, N, BASE |
| 102 | parser = argparse.ArgumentParser(description=__doc__) |
| 103 | parser.add_argument('--reference', required=True, help='Original PNG-based pack, not the EPK3 result') |
| 104 | parser.add_argument('--textures', required=True) |
| 105 | parser.add_argument('--output', default='emoji.pack') |
| 106 | parser.add_argument('--workers', type=int, default=4) |
| 107 | parser.add_argument('--cache', help='Optional trusted directory for resumable intermediate stages') |
| 108 | parser.add_argument('--lossless', action='store_true', help='Keep the old visual result exactly; hidden RGB may differ') |
| 109 | args = parser.parse_args() |
| 110 | if args.workers < 1: parser.error('--workers must be positive') |
| 111 | if Path(args.reference).resolve() == Path(args.output).resolve(): parser.error('Use a separate output path') |
| 112 | start = time.monotonic() |
| 113 | R, NE, lookup, original_size = old.read_reference(args.reference, args.textures) |
| 114 | N = len(R) |
| 115 | with zipfile.ZipFile(args.textures) as archive: |
| 116 | O = np.array([np.array(Image.open(archive.open(f"{d['id']//4096}_{d['id']%4096}.png")).convert('RGBA')) for d in R[:NE]]) |
| 117 | old.R = R; old.N = N; old.NE = NE; old.MASK_LOOKUP = lookup |
| 118 | old.IDX = old.I = np.array([d['idx'] for d in R]) |
| 119 | visual.initialize(R, O); BASE = visual.RGBA |
| 120 | fingerprint = hashlib.sha256() |
| 121 | for f in [Path(args.reference), Path(args.textures), Path(__file__), Path(old.__file__), Path(visual.__file__)]: |
| 122 | fingerprint.update(f.read_bytes()) |
| 123 | fingerprint.update((Image.__version__ + str(features.version('webp'))).encode()) |
| 124 | key = fingerprint.hexdigest() |
| 125 | cache = Path(args.cache) if args.cache else None |
| 126 | if cache: cache.mkdir(parents=True, exist_ok=True) |
| 127 | def stage(name, fn, jobs): |
| 128 | path = cache / (name + '.pkl') if cache else None |
| 129 | if path and path.exists(): |
| 130 | # This cache is produced by this tool. Do not use untrusted pickle files. |
| 131 | with path.open('rb') as stream: saved = pickle.load(stream) |
| 132 | if saved['key'] == key: |
| 133 | print(name + ': cache', flush=True); return saved['value'] |
| 134 | value = old.map_jobs(fn, jobs, args.workers, name) |
| 135 | if path: |
| 136 | temp = path.with_suffix('.tmp') |
| 137 | with temp.open('wb') as stream: pickle.dump({'key': key, 'value': value}, stream, protocol=5) |
| 138 | temp.replace(path) |
| 139 | return value |
| 140 | old.B = stage('base-webp', old.baseline, R) |
| 141 | old.C = old.candidates() |
| 142 | old.D = stage('base-delta', old.encode_delta, range(N)) |
| 143 | old.W = stage('base-delta-webp', old.encode_delta_webp, range(N)) |
| 144 | old.features = old.half_candidates() |
| 145 | base_half = stage('base-halves', old.encode_half, range(N)) |
| 146 | visual.make_candidates(); visual.D = old.D |
| 147 | mirrors = stage('mirrors', visual.mirror_job, range(N)) |
| 148 | templates = half_templates = native = [] |
| 149 | if not args.lossless: |
| 150 | templates = stage('fitted-palettes', visual.template_job, range(NE)) |
| 151 | visual.make_half_candidates() |
| 152 | half_templates = stage('fitted-halves', visual.half_template_job, range(NE)) |
| 153 | native = stage('native-webp', visual.native_webp_job, range(NE)) |
| 154 | selected = assemble(base_half, mirrors, templates, half_templates, native, args.lossless) |
| 155 | output = Path(args.output); output.parent.mkdir(parents=True, exist_ok=True) |
| 156 | data = bytearray(old.pack(selected, output)); struct.pack_into('<H', data, 4, 2) |
| 157 | output.write_bytes(data) |
| 158 | stats = dict(reference_bytes=original_size, previous_pack_bytes=3670504, pack_bytes=len(data), |
| 159 | saving_percent=100*(1-len(data)/original_size), |
| 160 | emoji_count=NE, mask_count=N-NE, |
| 161 | fitted_records=sum(o.get('approx',False) and 'native_webp_quality' not in o for o in selected), |
| 162 | native_lossy_records=sum('native_webp_quality' in o for o in selected), |
| 163 | horizontal_mirror_references=sum(t==1 for o in selected for t in o.get('transforms',())), |
| 164 | transformed_references=sum(t!=0 for o in selected for t in o.get('transforms',())), |
| 165 | mode_counts=dict(Counter(f"codec={o['codec']},prediction={o['pred']}" for o in selected)), |
| 166 | sha256=hashlib.sha256(data).hexdigest(),build_seconds=round(time.monotonic()-start,2)) |
| 167 | output.with_suffix('.stats.json').write_text(json.dumps(stats,indent=2)+'\n') |
| 168 | print(json.dumps(stats,indent=2),flush=True) |
| 169 | # Validate the actual serialized bytes and the completed RGBA, including private masks. |
| 170 | from verify_emoji_pack import validate |
| 171 | verification = validate(output, args.reference, args.textures) |
| 172 | output.with_suffix('.verification.json').write_text(json.dumps(verification,indent=2)+'\n') |
| 173 |
| 174 | if __name__ == '__main__': main() |
| 175 |