Deprecate old packs system and replace it with internal generated packs

get.php

@@ -1,6 +1,6 @@
 <?php
 /*
-Copyright 2018 UUP dump API authors
+Copyright 2019 UUP dump API authors
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -52,49 +52,22 @@ function uupGetFiles(
         $info['sku'] = 48;
     }
 
-    $packs = uupGetPacks($build);
-    $packsForLangs = $packs['packsForLangs'];
-    $editionPacks = $packs['editionPacks'];
-    $checkEditions = $packs['allEditions'];
-    $skipNeutral = $packs['skipNeutral'];
-    $skipLangPack = $packs['skipLangPack'];
-    $packs = $packs['packs'];
-
-    $noLangPack = 0;
-    $noNeutral = 0;
-
-    $useGeneratedPacks = 0;
-
-    if(file_exists('packs/'.$updateId.'.json.gz') && $usePack) {
-        $genPack = @gzdecode(@file_get_contents('packs/'.$updateId.'.json.gz'));
-
-        if(!empty($genPack)) {
-            $genPack = json_decode($genPack, 1);
+    if($usePack) {
+        $genPack = uupGetGenPacks($build, $info['arch'], $updateId);
+        if(empty($genPack)) return array('error' => 'UNSUPPORTED_COMBINATION');
 
         if(!isset($genPack[$usePack])) {
             return array('error' => 'UNSUPPORTED_LANG');
         }
-
-            $packsForLangs = array();
-            $packsForLangs[$usePack] = array(0);
-
-            $useGeneratedPacks = 1;
-        }
-    }
-
-    if($usePack) {
-        $usePack = strtolower($usePack);
-        if(!isset($packsForLangs[$usePack])) {
-            return array('error' => 'UNSUPPORTED_LANG');
-        }
     }
 
     $desiredEdition = strtoupper($desiredEdition);
+    $fileListSource = $desiredEdition;
 
     switch($desiredEdition) {
         case '0':
-            if($useGeneratedPacks) {
-                $desiredEdition = 'GENERATEDPACKS';
+            if($usePack) {
+                $fileListSource = 'GENERATEDPACKS';
 
                 $filesList = array();
                 foreach($genPack[$usePack] as $val) {
@@ -113,58 +86,15 @@ function uupGetFiles(
         case 'UPDATEONLY': break;
 
         default:
-            if($useGeneratedPacks) {
             if(!isset($genPack[$usePack][$desiredEdition])) {
                 return array('error' => 'UNSUPPORTED_COMBINATION');
             }
 
             $filesList = $genPack[$usePack][$desiredEdition];
-                $desiredEdition = 'GENERATEDPACKS';
+            $fileListSource = 'GENERATEDPACKS';
             break;
     }
 
-            if(!$usePack) {
-                return array('error' => 'UNSPECIFIED_LANG');
-            }
-
-            if(!isset($editionPacks[$desiredEdition])) {
-                return array('error' => 'UNSUPPORTED_EDITION');
-            }
-
-            $supported = 0;
-            foreach($packsForLangs[$usePack] as $val) {
-                if($editionPacks[$desiredEdition] == $val) $supported = 1;
-            }
-
-            if(!$supported) {
-                return array('error' => 'UNSUPPORTED_COMBINATION');
-            }
-            unset($supported);
-
-            if(isset($skipLangPack[$desiredEdition])) {
-                if($skipLangPack[$desiredEdition]) {
-                    $noLangPack = 1;
-                }
-            }
-
-            if(isset($skipNeutral[$desiredEdition])) {
-                if($skipNeutral[$desiredEdition]) {
-                    $noNeutral = 1;
-                }
-            }
-
-            $checkEditions = array($desiredEdition);
-            break;
-    }
-
-    if($noNeutral) {
-        foreach($packsForLangs[$usePack] as $num) {
-            if(isset($packs[$num]['editionNeutral'])) {
-                unset($packs[$num]['editionNeutral']);
-            }
-        }
-    }
-
     $cacheHash = hash('sha1', strtolower("api-get-$updateId"));
     $cached = 0;
 
@@ -328,7 +258,7 @@ function uupGetFiles(
 
     $filesKeys = array_keys($files);
 
-    switch($desiredEdition) {
+    switch($fileListSource) {
         case 'UPDATEONLY':
             $skipPackBuild = 1;
             $removeFiles = preg_grep('/Windows10\.0-KB.*-EXPRESS/i', $filesKeys);
@@ -350,73 +280,9 @@ function uupGetFiles(
             $skipPackBuild = 1;
             $filesKeys = preg_grep('/WindowsUpdateBox.exe/i', $filesKeys);
             break;
-
-        case 'GENERATEDPACKS':
-            $skipPackBuild = 1;
-            break;
-
-        default:
-            $skipPackBuild = 0;
-            break;
     }
 
-    if($usePack && !$skipPackBuild) {
-        $esd = array_keys($files);
-        $esd = preg_grep('/\.esd$/i', $esd);
-
-        foreach($esd as $key => $val) {
-            $esd[$key] = strtoupper($val);
-        }
-
-        foreach($checkEditions as $val) {
-            $testEsd[] = $val.'_'.strtoupper($usePack).'.ESD';
-        }
-
-        $foundMetadata = array_intersect($testEsd, $esd);
-        consoleLogger('Found '.count($foundMetadata).' metadata ESD file(s).');
-
-        if(empty($foundMetadata)) {
-            return array('error' => 'NO_METADATA_ESD');
-        }
-
-        $removeFiles = array();
-        $removeFiles[0] = preg_grep('/RetailDemo-OfflineContent/i', $filesKeys);
-        $removeFiles[1] = preg_grep('/Windows10\.0-KB.*-EXPRESS/i', $filesKeys);
-
-        foreach($removeFiles as $val) {
-            foreach($val as $temp) {
-                if(isset($files[$temp])) unset($files[$temp]);
-            }
-        }
-        unset($removeFiles, $temp, $val);
-
-        $filesKeys = array_keys($files);
-        $filesTemp = array();
-
-        if(!$noLangPack) {
-            $temp = preg_grep('/.*'.$usePack.'-Package.*/i', $filesKeys);
-            $filesTemp = array_merge($filesTemp, $temp);
-
-            $temp = preg_grep('/.*'.$usePack.'_lp..../i', $filesKeys);
-            $filesTemp = array_merge($filesTemp, $temp);
-        }
-
-        foreach($packsForLangs[$usePack] as $num) {
-            foreach($packs[$num] as $key => $val) {
-                if($key == 'editionNeutral'
-                || $key == $desiredEdition
-                || !$desiredEdition) {
-                    $temp = packsByEdition($key, $val, $usePack, $filesKeys);
-                    $filesTemp = array_merge($filesTemp, $temp);
-                }
-            }
-        }
-
-        $filesKeys = array_unique($filesTemp);
-        unset($filesTemp, $temp, $val, $num);
-    }
-
-    if($desiredEdition == 'GENERATEDPACKS') {
+    if($fileListSource == 'GENERATEDPACKS') {
         $temp = preg_grep('/Windows10\.0-KB.*-EXPRESS/i', $filesKeys, PREG_GREP_INVERT);
         $temp = preg_grep('/Windows10\.0-KB/i', $temp);
         $filesList = array_merge($filesList, $temp);
@@ -428,8 +294,10 @@ function uupGetFiles(
             $name = preg_replace('/~/', '-', $name);
             $name = strtolower($name);
 
+            if(isset($files[$name])) {
                 $newFiles[$name] = $files[$name];
             }
+        }
 
         $files = $newFiles;
         $filesKeys = array_keys($files);
@@ -456,20 +324,3 @@ function uupGetFiles(
         'files' => $files,
     );
 }
-
-function packsByEdition($edition, $pack, $lang, $filesKeys) {
-    $filesTemp = array();
-
-    if($edition != 'editionNeutral') {
-        $temp = preg_grep('/'.$edition.'_'.$lang.'\.esd/i', $filesKeys);
-        $filesTemp = array_merge($filesTemp, $temp);
-    }
-
-    foreach($pack as $val) {
-        $temp = preg_grep('/'.$val.'.*/i', $filesKeys);
-        $filesTemp = array_merge($filesTemp, $temp);
-    }
-
-    return $filesTemp;
-}
-?>

listeditions.php

@@ -1,6 +1,6 @@
 <?php
 /*
-Copyright 2017 UUP dump API authors
+Copyright 2019 UUP dump API authors
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -21,52 +21,36 @@ require_once dirname(__FILE__).'/updateinfo.php';
 
 function uupListEditions($lang = 'en-us', $updateId = 0) {
     if($updateId) {
-        $info = uupUpdateInfo($updateId, 'build');
+        $info = uupUpdateInfo($updateId);
     }
 
-    if(isset($info['info'])) {
-        $build = explode('.', $info['info']);
+    if(isset($info['info'])) $info = $info['info'];
+
+    if(isset($info['build'])) {
+        $build = explode('.', $info['build']);
         $build = $build[0];
     } else {
         $build = 9841;
     }
 
-    $packs = uupGetPacks($build);
-    $packsForLangs = $packs['packsForLangs'];
-    $fancyEditionNames = $packs['fancyEditionNames'];
-    $packs = $packs['packs'];
-
-    if(file_exists('packs/'.$updateId.'.json.gz')) {
-        $genPack = @gzdecode(@file_get_contents('packs/'.$updateId.'.json.gz'));
-
-        if(!empty($genPack)) {
-            $genPack = json_decode($genPack, 1);
-
-            if(!isset($genPack[$lang])) {
-                return array('error' => 'UNSUPPORTED_LANG');
+    if(!isset($info['arch'])) {
+        $arch = null;
     }
 
-            $packsForLangs = array();
-            $packsForLangs[$lang] = array(0);
-
-            $packs = array();
-            $packs[0] = $genPack[$lang];
-        }
-    }
+    $genPack = uupGetGenPacks($build, $info['arch'], $updateId);
+    $fancyTexts = uupGetInfoTexts();
+    $fancyEditionNames = $fancyTexts['fancyEditionNames'];
 
     if($lang) {
         $lang = strtolower($lang);
-        if(!isset($packsForLangs[$lang])) {
+        if(!isset($genPack[$lang])) {
             return array('error' => 'UNSUPPORTED_LANG');
         }
     }
 
     $editionList = array();
     $editionListFancy = array();
-    foreach($packsForLangs[$lang] as $val) {
-        foreach(array_keys($packs[$val]) as $edition) {
-            if($edition == 'editionNeutral') continue;
-
+    foreach(array_keys($genPack[$lang]) as $edition) {
         if(isset($fancyEditionNames[$edition])) {
             $fancyName = $fancyEditionNames[$edition];
         } else {
@@ -76,7 +60,6 @@ function uupListEditions($lang = 'en-us', $updateId = 0) {
         $editionList[] = $edition;
         $editionListFancy[$edition] = $fancyName;
     }
-    }
 
     return array(
         'apiVersion' => uupApiVersion(),
@@ -84,4 +67,3 @@ function uupListEditions($lang = 'en-us', $updateId = 0) {
         'editionFancyNames' => $editionListFancy,
     );
 }
-?>

listlangs.php

@@ -1,6 +1,6 @@
 <?php
 /*
-Copyright 2017 UUP dump API authors
+Copyright 2019 UUP dump API authors
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -21,40 +21,37 @@ require_once dirname(__FILE__).'/updateinfo.php';
 
 function uupListLangs($updateId = 0) {
     if($updateId) {
-        $info = uupUpdateInfo($updateId, 'build');
+        $info = uupUpdateInfo($updateId);
     }
 
-    if(isset($info['info'])) {
-        $build = explode('.', $info['info']);
+    if(isset($info['info'])) $info = $info['info'];
+
+    if(isset($info['build'])) {
+        $build = explode('.', $info['build']);
         $build = $build[0];
     } else {
         $build = 9841;
     }
 
-    $packs = uupGetPacks($build);
-    $packsForLangs = $packs['packsForLangs'];
-    $fancyLangNames = $packs['fancyLangNames'];
-
-    if(file_exists('packs/'.$updateId.'.json.gz')) {
-        $genPack = @gzdecode(@file_get_contents('packs/'.$updateId.'.json.gz'));
-
-        if(!empty($genPack)) {
-            $genPack = json_decode($genPack, 1);
-            $packsForLangs = $genPack;
-        }
+    if(!isset($info['arch'])) {
+        $arch = null;
     }
 
+    $genPack = uupGetGenPacks($build, $info['arch'], $updateId);
+    $fancyTexts = uupGetInfoTexts();
+    $fancyLangNames = $fancyTexts['fancyLangNames'];
+
     $langList = array();
     $langListFancy = array();
-    foreach($packsForLangs as $key => $val) {
-        if(isset($fancyLangNames[$key])) {
-            $fancyName = $fancyLangNames[$key];
+    foreach(array_keys($genPack) as $val) {
+        if(isset($fancyLangNames[$val])) {
+            $fancyName = $fancyLangNames[$val];
         } else {
-            $fancyName = $key;
+            $fancyName = $val;
         }
 
-        $langList[] = $key;
-        $langListFancy[$key] = $fancyName;
+        $langList[] = $val;
+        $langListFancy[$val] = $fancyName;
     }
 
     return array(
@@ -63,4 +60,3 @@ function uupListLangs($updateId = 0) {
         'langFancyNames' => $langListFancy,
     );
 }
-?>

shared/main.php

@@ -1,6 +1,6 @@
 <?php
 /*
-Copyright 2018 UUP dump API authors
+Copyright 2019 UUP dump API authors
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -16,7 +16,7 @@ limitations under the License.
 */
 
 function uupApiVersion() {
-    return '1.17.3';
+    return '1.18.0';
 }
 
 require_once dirname(__FILE__).'/auths.php';

shared/packs.php

@@ -15,7 +15,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-function uupGetPacks($build = 15063) {
+require_once dirname(__FILE__).'/../listid.php';
+
+function uupGetInfoTexts() {
     $fancyLangNames = array(
         'ar-sa' => 'Arabic (Saudi Arabia)',
         'bg-bg' => 'Bulgarian',
@@ -61,7 +63,7 @@ function uupGetPacks($build = 15063) {
     $fancyEditionNames = array(
         'CLOUD' => 'Windows 10 S',
         'CLOUDN' => 'Windows 10 S N',
-        'CLOUDE' => 'Windows 10 CloudE',
+        'CLOUDE' => 'Windows 10 Lean',
         'CORE' => 'Windows 10 Home',
         'CORECOUNTRYSPECIFIC' => 'Windows 10 Home China',
         'COREN' => 'Windows 10 Home N',
@@ -165,33 +167,171 @@ function uupGetPacks($build = 15063) {
         'STARTERN',
     );
 
-    if($build < 17063) {
-        require dirname(__FILE__).'/packs/legacy.php';
-    } elseif ($build >= 17661) {
-        require dirname(__FILE__).'/packs/17661.php';
-    } elseif ($build >= 17655) {
-        require dirname(__FILE__).'/packs/17655.php';
-    } elseif ($build >= 17650) {
-        require dirname(__FILE__).'/packs/17650.php';
-    } elseif ($build >= 17634) {
-        require dirname(__FILE__).'/packs/17634.php';
-    } elseif ($build >= 17623) {
-        require dirname(__FILE__).'/packs/17623.php';
-    } elseif ($build >= 17093) {
-        require dirname(__FILE__).'/packs/17093.php';
-    } elseif ($build >= 17063) {
-        require dirname(__FILE__).'/packs/17063.php';
-    }
-
     return array(
-        'packs' => $packs,
-        'packsForLangs' => $packsForLangs,
-        'editionPacks' => $editionPacks,
         'fancyEditionNames' => $fancyEditionNames,
         'fancyLangNames' => $fancyLangNames,
         'allEditions' => $allEditions,
-        'skipNeutral' => $skipNeutral,
-        'skipLangPack' => $skipLangPack,
     );
 }
-?>
+
+function uupGetGenPacks($build = 15063, $arch = null, $updateId = null) {
+    $internalPacks = dirname(__FILE__).'/packs';
+
+    if(!file_exists($internalPacks.'/metadata.json')) {
+        if(!uupCreateInternalPacksMetadata($internalPacks)) {
+            return array();
+        }
+    }
+
+    if(!empty($updateId)) {
+        if(file_exists('packs/'.$updateId.'.json.gz')) {
+            $genPack = @gzdecode(@file_get_contents('packs/'.$updateId.'.json.gz'));
+            if(empty($genPack)) return array();
+
+            $genPack = json_decode($genPack, 1);
+            return $genPack;
+        }
+    }
+
+    $metadata = @file_get_contents($internalPacks.'/metadata.json');
+    if(empty($metadata)) {
+        return array();
+    } else {
+        $metadata = json_decode($metadata, 1);
+    }
+
+    $hashDetermined = 0;
+    $useAllHashesForBuild = 0;
+
+    if($updateId) {
+        if(isset($metadata['knownIds'][$updateId])) {
+            $hash = $metadata['knownIds'][$updateId];
+            $hashDetermined = 1;
+        }
+    }
+
+    if(!$hashDetermined) {
+        foreach($metadata['knownBuilds'] as $buildNum => $val) {
+            if($build < $buildNum) continue;
+            $useBuild = $buildNum;
+            break;
+        }
+
+        if(!isset($useBuild)) {
+            return array();
+        }
+
+        if(!$arch && !isset($metadata['knownBuilds'][$useBuild][$arch])) {
+            $genPack = array();
+            foreach($metadata['knownBuilds'][$useBuild] as $hash) {
+                $temp = @gzdecode(@file_get_contents($internalPacks.'/'.$hash.'.json.gz'));
+                if(!empty($temp)) {
+                    $temp = json_decode($temp, 1);
+                    $genPack = array_merge_recursive($genPack, $temp);
+                    unset($temp);
+                }
+            }
+        } else {
+            $hash = $metadata['knownBuilds'][$useBuild][$arch];
+        }
+    }
+
+    if(!isset($genPack)) {
+        $genPack = @gzdecode(@file_get_contents($internalPacks.'/'.$hash.'.json.gz'));
+        if(!empty($genPack)) {
+            $genPack = json_decode($genPack, 1);
+        } else {
+            $genPack = array();
+        }
+    }
+
+    return $genPack;
+}
+
+//Function to regenerate internal packs. Should not be used when not needed.
+function uupCreateInternalPacksMetadata($internalPacks) {
+    $metadataCreationAllowed = 0;
+    if(!$metadataCreationAllowed) return false;
+
+    $builds = uupListIds();
+    if(isset($ids['error'])) {
+        return false;
+    }
+
+    $builds = $builds['builds'];
+
+    if(!file_exists('packs')) return false;
+
+    if(!file_exists($internalPacks)) {
+        if(!mkdir($internalPacks)) {
+            return false;
+        }
+    } else {
+        rmdir($internalPacks);
+        mkdir($internalPacks);
+    }
+
+    $files = scandir('packs');
+    $files = preg_grep('/\.json.gz$/', $files);
+
+    $packs = array();
+    foreach($builds as $build) {
+        $uuid = $build['uuid'];
+        $file = $uuid.'.json.gz';
+
+        if(!file_exists('packs/'.$file)) continue;
+
+        $genPack = @gzdecode(@file_get_contents('packs/'.$file));
+        $hash = hash('sha1', $genPack);
+
+        if(!file_exists($internalPacks.'/'.$hash.'.json.gz')) {
+            if(!copy('packs/'.$file, $internalPacks.'/'.$hash.'.json.gz')) {
+                return false;
+            }
+        }
+
+        $packs['knownIds'][$uuid] = $hash;
+
+        $buildNum = explode('.', $build['build']);
+        $buildNum = $buildNum[0];
+
+        $packs['knownBuilds'][$buildNum][$build['arch']] = $hash;
+    }
+
+    file_put_contents($internalPacks.'/metadata.json', json_encode($packs)."\n");
+    return true;
+}
+
+//Emulation of legacy packs. Do not use in new scripts due to extremely slow process.
+function uupGetPacks($build = 15063) {
+    $returnArray = uupGetInfoTexts();
+    $genPack = uupGetGenPacks($build);
+
+    foreach($genPack as $lang => $editions) {
+        $packsForLangs[$lang] = array_keys($editions);
+        $packsForLangs[$lang][] = $lang;
+
+        foreach(array_keys($editions) as $edition) {
+            foreach($editions[$edition] as $name) {
+                $newName = preg_replace('/^cabs_|^metadataesd_|~31bf3856ad364e35/i', '', $name);
+                $newName = preg_replace('/~~\.|~\./', '.', $newName);
+                $newName = preg_replace('/~/', '-', $newName);
+                $newName = strtolower($newName);
+                $packs[$lang][$edition][] = $newName;
+            }
+
+            $editionPacks[$edition] = $edition;
+            $packs[$edition][$edition] = array();
+            $skipNeutral[$edition] = 1;
+            $skipLangPack[$edition] = 1;
+        }
+    }
+
+    $returnArray['packs'] = $packs;
+    $returnArray['packsForLangs'] = $packsForLangs;
+    $returnArray['editionPacks'] = $editionPacks;
+    $returnArray['skipNeutral'] = $skipNeutral;
+    $returnArray['skipLangPack'] = $skipLangPack;
+
+    return $returnArray;
+}