Rewrite WU requests and cookie handling

v1.40.3

.gitignore

@@ -1 +0,0 @@
-shared/cookie.json

fetchupd.php

@@ -17,6 +17,8 @@ limitations under the License.
 
 require_once dirname(__FILE__).'/shared/main.php';
 require_once dirname(__FILE__).'/shared/requests.php';
+require_once dirname(__FILE__).'/shared/cache.php';
+require_once dirname(__FILE__).'/shared/fileinfo.php';
 require_once dirname(__FILE__).'/listid.php';
 
 function uupFetchUpd(
@@ -37,14 +39,18 @@ function uupFetchUpd(
     $flight = 'Active';
 
     if($build == 'latest' || (!$build)) {
-        $builds = array('17134.1');
+        $builds = array('22000.1');
 
         $ids = uupListIds();
         if(isset($ids['error'])) {
             $ids['builds'] = array();
         }
 
-        $build = $ids['builds'][0]['build'];
+        if(empty($ids['builds'])) {
+            $build = $builds[0];
+        } else {
+            $build = $ids['builds'][0]['build'];
+        }
         unset($builds, $ids);
     }
 
@@ -57,7 +63,7 @@ function uupFetchUpd(
         return array('error' => 'UNKNOWN_ARCH');
     }
 
-    if(!($ring == 'DEV' || $ring == 'BETA' || $ring == 'RELEASEPREVIEW' || $ring == 'WIF' || $ring == 'WIS' || $ring == 'RP' || $ring == 'RETAIL' || $ring == 'MSIT')) {
+    if(!($ring == 'CANARY' || $ring == 'DEV' || $ring == 'BETA' || $ring == 'RELEASEPREVIEW' || $ring == 'WIF' || $ring == 'WIS' || $ring == 'RP' || $ring == 'RETAIL' || $ring == 'MSIT')) {
         return array('error' => 'UNKNOWN_RING');
     }
 
@@ -90,44 +96,21 @@ function uupFetchUpd(
         $type = 'Production';
     }
 
-    $cacheHash = hash('sha256', strtolower("api-fetch-$arch-$ring-$flight-$build-$minor-$sku-$type"));
-    $cached = 0;
+    $res = "api-fetch-$arch-$ring-$flight-$build-$minor-$sku-$type";
+    $cache = new UupDumpCache($res);
+    $fromCache = $cache->get();
+    if($fromCache !== false) return $fromCache;
 
-    if(file_exists('cache/'.$cacheHash.'.json.gz') && $cacheRequests == 1) {
-        $cache = @gzdecode(@file_get_contents('cache/'.$cacheHash.'.json.gz'));
-        $cache = json_decode($cache, 1);
-
-        if(!empty($cache['content']) && ($cache['expires'] > time())) {
-            consoleLogger('Using cached response...');
-            $out = $cache['content'];
-            $cached = 1;
-        } else {
-            $cached = 0;
-        }
-
-        unset($cache);
+    consoleLogger('Fetching information from the server...');
+    $composerArgs = [$arch, $flight, $ring, $build, $sku, $type];
+    $out = sendWuPostRequestHelper('client', 'composeFetchUpdRequest', $composerArgs);
+    if($out['error'] != 200) {
+        consoleLogger('The request has failed');
+        return array('error' => 'WU_REQUEST_FAILED');
     }
 
-    if(!$cached) {
-        consoleLogger('Fetching information from the server...');
-        $postData = composeFetchUpdRequest(uupDevice(), uupEncryptedData(), $arch, $flight, $ring, $build, $sku, $type);
-        $out = sendWuPostRequest('https://fe3cr.delivery.mp.microsoft.com/ClientWebService/client.asmx', $postData);
-
-        $out = html_entity_decode($out);
-        consoleLogger('Information has been successfully fetched.');
-
-        if($cacheRequests == 1) {
-            $cache = array(
-                'expires' => time()+120,
-                'content' => $out,
-            );
-
-            if(!file_exists('cache')) mkdir('cache');
-            @file_put_contents('cache/'.$cacheHash.'.json.gz', gzencode(json_encode($cache)."\n"));
-
-            unset($cache);
-        }
-    }
+    $out = html_entity_decode($out['out']);
+    consoleLogger('Information has been successfully fetched.');
 
     preg_match_all('/<UpdateInfo>.*?<\/UpdateInfo>/', $out, $updateInfos);
     $updateInfo = preg_grep('/<IsLeaf>true<\/IsLeaf>/', $updateInfos[0]);
@@ -160,7 +143,7 @@ function uupFetchUpd(
         return array('error' => 'EMPTY_FILELIST');
     }
 
-    return array(
+    $data = [
         'apiVersion' => uupApiVersion(),
         'updateId' => $updateArray[0]['updateId'],
         'updateTitle' => $updateArray[0]['updateTitle'],
@@ -168,7 +151,13 @@ function uupFetchUpd(
         'arch' => $updateArray[0]['arch'],
         'fileWrite' => $updateArray[0]['fileWrite'],
         'updateArray' => $updateArray,
-    );
+    ];
+
+    if($cacheRequests == 1) {
+        $cache->put($data, 120);
+    }
+
+    return $data;
 }
 
 function parseFetchUpdate($updateInfo, $out, $arch, $ring, $flight, $build, $sku, $type) {
@@ -237,8 +226,10 @@ function parseFetchUpdate($updateInfo, $out, $arch, $ring, $flight, $build, $sku
 
     $updateTitle = preg_replace("/ ?\d{4}-\d{2}\D ?| ?$foundArch ?| ?x64 ?/i", '', $updateTitle);
 
-    if($foundType == 'server')
+    if($foundType == 'server') {
         $updateTitle = str_replace('Windows 10', 'Windows Server', $updateTitle);
+        $updateTitle = str_replace('Windows 11', 'Windows Server', $updateTitle);
+    }
 
     if($sku == 406)
         $updateTitle = str_replace('Microsoft server operating system', 'Azure Stack HCI', $updateTitle);
@@ -300,16 +291,16 @@ function parseFetchUpdate($updateInfo, $out, $arch, $ring, $flight, $build, $sku
         return array('error' => 'BROKEN_UPDATE');
     }
 
-    if(preg_match('/Corpnet Required/i', $updateTitle)) {
+    $isCorpnet = preg_match('/Corpnet Required/i', $updateTitle);
+    if($isCorpnet && !uupApiConfigIsTrue('allow_corpnet')) {
         consoleLogger('Skipping corpnet only update...');
         return array('error' => 'CORPNET_ONLY_UPDATE');
     }
 
     $fileWrite = 'NO_SAVE';
-    if(!file_exists('fileinfo/'.$updateString.'.json')) {
+    if(!uupApiFileInfoExists($updateId)) {
         consoleLogger('WARNING: This build is NOT in the database. It will be saved now.');
         consoleLogger('Parsing information to write...');
-        if(!file_exists('fileinfo')) mkdir('fileinfo');
 
         $fileList = preg_replace('/<Files>|<\/Files>/', '', $fileList[0]);
         preg_match_all('/<File.*?<\/File>/', $fileList, $fileList);
@@ -367,10 +358,11 @@ function parseFetchUpdate($updateInfo, $out, $arch, $ring, $flight, $build, $sku
         consoleLogger('Successfully parsed the information.');
         consoleLogger('Writing new build information to the disk...');
 
-        $success = file_put_contents('fileinfo/'.$updateString.'.json', json_encode($temp)."\n");
+        $success = uupApiWriteFileinfo($updateString, $temp);
         if($success) {
             consoleLogger('Successfully written build information to the disk.');
             $fileWrite = 'INFO_WRITTEN';
+            uupApiPrivateInvalidateFileinfoCache();
         } else {
             consoleLogger('An error has occured while writing the information to the disk.');
         }

get.php

@@ -18,6 +18,8 @@ limitations under the License.
 require_once dirname(__FILE__).'/shared/main.php';
 require_once dirname(__FILE__).'/shared/requests.php';
 require_once dirname(__FILE__).'/shared/packs.php';
+require_once dirname(__FILE__).'/shared/cache.php';
+require_once dirname(__FILE__).'/shared/fileinfo.php';
 
 /*
 $updateId       = Update Identifier
@@ -42,10 +44,16 @@ function uupGetFiles(
     }
 
     if(!uupApiCheckUpdateId($updateId)) {
-            return array('error' => 'INCORRECT_ID');
+        return array('error' => 'INCORRECT_ID');
     }
 
-    $info = @file_get_contents('fileinfo/'.$updateId.'.json');
+    $edition = is_array($desiredEdition) ? implode('_', $desiredEdition) : $desiredEdition;
+    $res = "api-get-{$updateId}_{$usePack}_{$edition}_{$requestType}";
+    $cache = new UupDumpCache($res);
+    $fromCache = $cache->get();
+    if($fromCache !== false) return $fromCache;
+
+    $info = uupApiReadFileinfo($updateId);
     if(empty($info)) {
         $info = array(
             'ring' => 'WIF',
@@ -55,8 +63,6 @@ function uupGetFiles(
             'sku' => '48',
             'files' => array(),
         );
-    } else {
-        $info = json_decode($info, true);
     }
 
     if(isset($info['build'])) {
@@ -79,6 +85,8 @@ function uupGetFiles(
         }
     }
 
+    $appEdition = 0;
+
     if(!is_array($desiredEdition)) {
         $desiredEdition = strtoupper($desiredEdition);
         $fileListSource = $desiredEdition;
@@ -104,13 +112,19 @@ function uupGetFiles(
 
             case 'UPDATEONLY': break;
 
+            case 'APP': $appEdition = 1;
+            case 'APP_MOMENT': $appEdition = 1;
+
             default:
                 if(!isset($genPack[$usePack][$desiredEdition])) {
                     return array('error' => 'UNSUPPORTED_COMBINATION');
                 }
 
-                $filesPacksList = $genPack[$usePack][$desiredEdition];
                 $fileListSource = 'GENERATEDPACKS';
+                $filesPacksList = $genPack[$usePack][$desiredEdition];
+                if($desiredEdition == 'APP' && isset($genPack[$usePack]['APP_MOMENT'])) {
+                    $filesPacksList = array_merge($filesPacksList, $genPack[$usePack]['APP_MOMENT']);
+                }
                 break;
         }
     } else {
@@ -123,6 +137,7 @@ function uupGetFiles(
                 return array('error' => 'UNSUPPORTED_COMBINATION');
             }
 
+            if($edition == 'APP' || $edition == 'APP_MOMENT') $appEdition = 1;
             $filesPacksList = array_merge($filesPacksList, $genPack[$usePack][$edition]);
         }
     }
@@ -138,6 +153,7 @@ function uupGetFiles(
     $updateBuild = (isset($info['build'])) ? $info['build'] : 'UNKNOWN';
     $updateName = (isset($info['title'])) ? $info['title'] : 'Unknown update: '.$updateId;
     $sha256capable = isset($info['sha256ready']);
+    $hasUpdates = false;
 
     if(isset($info['releasetype'])) {
         $type = $info['releasetype'];
@@ -164,63 +180,101 @@ function uupGetFiles(
         if(isset($filesInfoList[$val])) unset($filesInfoList[$val]);
     }
 
-    if(!$sha256capable) {
-        $baseless = preg_grep('/^baseless_|-baseless\....$/i', array_keys($filesInfoList));
-        foreach($baseless as $val) {
-            if(isset($filesInfoList[$val])) unset($filesInfoList[$val]);
-        }
-
-        $psf = array_keys($filesInfoList);
-        $psf = preg_grep('/\.psf$/i', $psf);
-
-        $psfk = preg_grep('/Windows10\.0-KB.*/i', $psf);
-        $psfk = preg_grep('/.*-EXPRESS|.*-baseless/i', $psfk, PREG_GREP_INVERT);
-        if($build > 21380) foreach($psfk as $key => $val) {
-            if(isset($psf[$key])) unset($psf[$key]);
-        }
-        unset($psfk);
-
-        $removeFiles = array();
-        foreach($psf as $val) {
-            $name = preg_replace('/\.psf$/i', '', $val);
-            $removeFiles[] = $name;
-            unset($filesInfoList[$val]);
-        }
-        unset($index, $name, $psf);
-
-        $temp = preg_grep('/'.$updateArch.'_.*|arm64\.arm_.*|arm64\.x86_.*/i', $removeFiles);
-        foreach($temp as $key => $val) {
-            if(isset($filesInfoList[$val.'.cab'])) unset($filesInfoList[$val.'.cab']);
-            unset($removeFiles[$key]);
-        }
-        unset($temp);
-
-        foreach($removeFiles as $val) {
-            if(isset($filesInfoList[$val.'.esd'])) {
-                if(isset($filesInfoList[$val.'.cab'])) unset($filesInfoList[$val.'.cab']);
-            }
-        }
-        unset($removeFiles);
+    $baseless = preg_grep('/^baseless_/i', array_keys($filesInfoList));
+    foreach($baseless as $val) {
+        if(isset($filesInfoList[$val])) unset($filesInfoList[$val]);
     }
 
+    $expresscab = preg_grep('/Windows(10|11)\.0-KB.*-EXPRESS|SSU-.*-EXPRESS/i', array_keys($filesInfoList));
+
+    $expresspsf = array();
+    foreach($expresscab as $val) {
+        $name = preg_replace('/-EXPRESS.cab$/i', '', $val);
+        $expresspsf[] = $name;
+        if(isset($filesInfoList[$val])) unset($filesInfoList[$val]);
+    }
+    unset($index, $name, $expresscab);
+
+    foreach($expresspsf as $val) {
+        if(isset($filesInfoList[$val.'.cab'])) {
+            if(isset($filesInfoList[$val.'.psf'])) unset($filesInfoList[$val.'.psf']);
+        }
+    }
+    unset($expresspsf);
+
+    $psf = array_keys($filesInfoList);
+    $psf = preg_grep('/\.psf$/i', $psf);
+
+    $psfk = preg_grep('/Windows(10|11)\.0-KB.*/i', $psf);
+    $psfk = preg_grep('/.*-EXPRESS/i', $psfk, PREG_GREP_INVERT);
+    if($build < 17763) $psfk = preg_grep('/Windows(10|11)\.0-KB.*_\d\.psf$/i', $psfk, PREG_GREP_INVERT);
+    foreach($psfk as $key => $val) {
+        if(isset($psf[$key])) unset($psf[$key]);
+    }
+    unset($psfk);
+
+    $removeFiles = array();
+    foreach($psf as $val) {
+        $name = preg_replace('/\.psf$/i', '', $val);
+        $removeFiles[] = $name;
+        unset($filesInfoList[$val]);
+    }
+    unset($index, $name, $psf);
+
+    $temp = preg_grep('/'.$updateArch.'_.*|arm64\.arm_.*|arm64\.x86_.*/i', $removeFiles);
+    foreach($temp as $key => $val) {
+        if(isset($filesInfoList[$val.'.cab'])) unset($filesInfoList[$val.'.cab']);
+        unset($removeFiles[$key]);
+    }
+    unset($temp);
+
+    foreach($removeFiles as $val) {
+        if(isset($filesInfoList[$val.'.esd'])) {
+            if(isset($filesInfoList[$val.'.cab'])) unset($filesInfoList[$val.'.cab']);
+        }
+    }
+    unset($removeFiles);
+
+    $msu = array_keys($filesInfoList);
+    $msu = preg_grep('/\.msu$/i', $msu);
+    $removeMSUs = array();
+    foreach($msu as $val) {
+        $name = preg_replace('/\.msu$/i', '', $val);
+        $removeMSUs[] = $name;
+    }
+    unset($index, $name, $msu);
+
     $filesInfoKeys = array_keys($filesInfoList);
+    $updatesRegex = '/Windows(10|11)\.0-KB|SSU-.*?\....$/i';
 
     switch($fileListSource) {
         case 'UPDATEONLY':
             $skipPackBuild = 1;
-            $removeFiles = preg_grep('/Windows10\.0-KB.*-EXPRESS|Windows10\.0-KB.*-baseless|SSU-\d*?\.\d*?-.{3,5}-EXPRESS/i', $filesInfoKeys);
+            $removeFiles = preg_grep('/Windows(10|11)\.0-KB.*-baseless/i', $filesInfoKeys);
 
             foreach($removeFiles as $val) {
                 if(isset($filesInfoList[$val])) unset($filesInfoList[$val]);
             }
             unset($removeFiles);
 
-            $filesInfoKeys = array_keys($filesInfoList);
+            foreach($removeMSUs as $val) {
+                if(isset($filesInfoList[$val.'.cab']) && isset($filesInfoList[$val.'.msu'])) {
+                    unset($filesInfoList[$val.'.msu']);
+                }
+            }
+            unset($removeMSUs);
 
-            $filesInfoKeys = preg_grep('/Windows10\.0-KB|SSU-\d*?\.\d*?-.*?\.cab/i', $filesInfoKeys);
+            $filesInfoKeys = array_keys($filesInfoList);
+            $temp = preg_grep('/.*?AggregatedMetadata.*?\.cab|.*?DesktopDeployment.*?\.cab/i', $filesInfoKeys);
+
+            $filesInfoKeys = preg_grep($updatesRegex, $filesInfoKeys);
             if(count($filesInfoKeys) == 0) {
                 return array('error' => 'NOT_CUMULATIVE_UPDATE');
             }
+
+            if($build > 21380) $filesInfoKeys = array_merge($filesInfoKeys, $temp);
+            unset($temp);
+            $hasUpdates = true;
             break;
 
         case 'WUBFILE':
@@ -234,8 +288,24 @@ function uupGetFiles(
     if($updateSku == 135) $uupCleanFunc = 'uupCleanHolo';
 
     if($fileListSource == 'GENERATEDPACKS') {
-        $temp = preg_grep('/Windows10\.0-KB.*-EXPRESS|Windows10\.0-KB.*-baseless|SSU-\d*?\.\d*?-.{3,5}-EXPRESS/i', $filesInfoKeys, PREG_GREP_INVERT);
-        $temp = preg_grep('/Windows10\.0-KB|SSU-\d*?\.\d*?-.*?\.cab/i', $temp);
+        foreach($removeMSUs as $val) {
+            if(isset($filesInfoList[$val.'.cab']) && isset($filesInfoList[$val.'.msu'])) {
+                unset($filesInfoList[$val.'.msu']);
+            }
+        }
+        unset($removeMSUs);
+        $filesInfoKeys = array_keys($filesInfoList);
+
+        $temp = preg_grep('/Windows(10|11)\.0-KB.*-baseless/i', $filesInfoKeys, PREG_GREP_INVERT);
+        if($appEdition) {
+            $temp = preg_grep('/.*?AggregatedMetadata.*?\.cab|.*?DesktopDeployment.*?\.cab/i', $temp);
+        } else if($build > 21380) {
+            $temp = preg_grep('/Windows(10|11)\.0-KB|SSU-.*?\....$|.*?AggregatedMetadata.*?\.cab|.*?DesktopDeployment.*?\.cab/i', $temp);
+        } else {
+            $temp = preg_grep($updatesRegex, $temp);
+        }
+
+        $hasUpdates = !empty(preg_grep($updatesRegex, $temp));
         $filesPacksList = array_merge($filesPacksList, $temp);
 
         $newFiles = array();
@@ -255,7 +325,7 @@ function uupGetFiles(
                     $newFiles[$name] = $filesInfoList[$name];
                 } else {
                     $failedFile = true;
-					consoleLogger("Missing file: $val");
+                    consoleLogger("Missing file: $val");
                 }
             }
         } else {
@@ -267,7 +337,7 @@ function uupGetFiles(
                     $newFiles[$name] = $filesInfoList[$name];
                 } else {
                     $failedFile = true;
-					consoleLogger("Missing file: $name");
+                    consoleLogger("Missing file: $name");
                 }
             }
         }
@@ -287,6 +357,7 @@ function uupGetFiles(
     $filesNew = array();
     foreach($filesInfoKeys as $val) {
        $filesNew[$val] = $filesInfoList[$val];
+       $filesNew[$val]['url'] = uupApiFixDownloadLink($filesInfoList[$val]['url']);
     }
 
     $files = $filesNew;
@@ -294,48 +365,53 @@ function uupGetFiles(
 
     consoleLogger('Successfully parsed the information.');
 
-    return array(
+    $data = [
         'apiVersion' => uupApiVersion(),
         'updateName' => $updateName,
         'arch' => $updateArch,
         'build' => $updateBuild,
         'sku' => $updateSku,
+        'hasUpdates' => $hasUpdates,
         'files' => $files,
-    );
+    ];
+
+    if($requestType > 0) {
+        $cacheData = $data;
+        $cache->put($cacheData, 30);
+    }
+
+    return $data;
 }
 
 function uupGetOnlineFiles($updateId, $rev, $info, $cacheRequests, $type) {
-    $cacheHash = hash('sha256', strtolower("api-get-${updateId}_rev.$rev"));
-    $cached = 0;
+    $res = "api-get-online-{$updateId}_rev.$rev";
+    $cache = new UupDumpCache($res);
+    $fromCache = $cache->get();
+    $cached = ($fromCache !== false);
 
-    if(file_exists('cache/'.$cacheHash.'.json.gz') && $cacheRequests == 1) {
-        $cache = @gzdecode(@file_get_contents('cache/'.$cacheHash.'.json.gz'));
-        $cache = json_decode($cache, 1);
-
-        if(!empty($cache['content']) && ($cache['expires'] > time())) {
-            consoleLogger('Using cached response...');
-            $out = $cache['content'];
-            $fetchTime = $cache['fetchTime'];
-            $cached = 1;
-        } else {
-            $cached = 0;
-        }
-
-        unset($cache);
-    }
-
-    if(!$cached) {
+    if($cached) {
+        $out = $fromCache['out'];
+        $fetchTime = $fromCache['fetchTime'];
+    } else {
         $fetchTime = time();
         consoleLogger('Fetching information from the server...');
-        $postData = composeFileGetRequest($updateId, uupDevice(), $info, $rev, $type);
-        $out = sendWuPostRequest('https://fe3cr.delivery.mp.microsoft.com/ClientWebService/client.asmx/secured', $postData);
+
+        $composerArgs = [$updateId, $info, $rev, $type];
+        $out = sendWuPostRequestHelper('clientSecured', 'composeFileGetRequest', $composerArgs);
+
+        if($out['error'] != 200) {
+            consoleLogger('The request has failed');
+            return array('error' => 'WU_REQUEST_FAILED');
+        }
+
+        $out = $out['out'];
         consoleLogger('Information has been successfully fetched.');
     }
 
     consoleLogger('Parsing information...');
     $xmlOut = @simplexml_load_string($out);
     if($xmlOut === false) {
-        @unlink('cache/'.$cacheHash.'.json.gz');
+        $cache->delete();
         return array('error' => 'XML_PARSE_ERROR');
     }
 
@@ -381,9 +457,19 @@ function uupGetOnlineFiles($updateId, $rev, $info, $cacheRequests, $type) {
         }
 
         if($sha256capable) {
-            $n = strrpos($name, '.');
-            if($n === false) $n = strlen($name);
-            $newName = substr($name, 0, $n).'_'.substr($sha1, 0, 8).substr($name, $n);
+            $tempname = uupCleanSha256($name);
+            if(isset($files[$tempname])) {
+                if($size > $files[$tempname]['size']) {
+                    $smaller = uupAppendSha1($tempname, $files[$tempname]['sha1']);
+                    $files[$smaller] = $files[$tempname];
+                    unset($files[$tempname]);
+                    $newName = $tempname;
+                } else {
+                    $newName = uupAppendSha1($tempname, $sha1);
+                }
+            } else {
+                $newName = $tempname;
+            }
         } else {
             $newName = $uupCleanFunc($name);
         }
@@ -421,14 +507,12 @@ function uupGetOnlineFiles($updateId, $rev, $info, $cacheRequests, $type) {
     }
 
     if($cacheRequests == 1 && $cached == 0) {
-        $cache = array(
-            'expires' => time()+90,
-            'content' => $out,
+        $cacheData = [
+            'out' => $out,
             'fetchTime' => $fetchTime,
-        );
+        ];
 
-        if(!file_exists('cache')) mkdir('cache');
-        @file_put_contents('cache/'.$cacheHash.'.json.gz', gzencode(json_encode($cache)."\n"));
+        $cache->put($cacheData, 90);
     }
 
     return $files;
@@ -450,9 +534,19 @@ function uupGetOfflineFiles($info) {
         $sha256 = isset($val['sha256']) ? $val['sha256'] : null;
 
         if($sha256capable) {
-            $n = strrpos($name, '.');
-            if($n === false) $n = strlen($name);
-            $newName = substr($name, 0, $n).'_'.substr($sha1, 0, 8).substr($name, $n);
+            $tempname = uupCleanSha256($name);
+            if(isset($files[$tempname])) {
+                if($size > $files[$tempname]['size']) {
+                    $smaller = uupAppendSha1($tempname, $files[$tempname]['sha1']);
+                    $files[$smaller] = $files[$tempname];
+                    unset($files[$tempname]);
+                    $newName = $tempname;
+                } else {
+                    $newName = uupAppendSha1($tempname, $sha1);
+                }
+            } else {
+                $newName = $tempname;
+            }
         } else {
             $newName = $uupCleanFunc($name);
         }
@@ -478,6 +572,24 @@ function uupGetOfflineFiles($info) {
     return $files;
 }
 
+function uupAppendSha1($name, $sha1) {
+    $n = strrpos($name, '.');
+    if($n === false) $n = strlen($name);
+    return substr($name, 0, $n).'_'.substr($sha1, 0, 8).substr($name, $n);
+}
+
+function uupCleanSha256($name) {
+    $replace = array(
+        'prss_signed_appx_' => null,
+        '~31bf3856ad364e35' => null,
+        '~~.' => '.',
+        '~.' => '.',
+        '~' => '-',
+    );
+
+    return strtr($name, $replace);
+}
+
 function uupCleanName($name) {
     $replace = array(
         'cabs_' => null,

listeditions.php

@@ -1,6 +1,6 @@
 <?php
 /*
-Copyright 2019 UUP dump API authors
+Copyright 2022 UUP dump API authors
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -21,7 +21,7 @@ require_once dirname(__FILE__).'/updateinfo.php';
 
 function uupListEditions($lang = 'en-us', $updateId = 0) {
     if($updateId) {
-        $info = uupUpdateInfo($updateId);
+        $info = uupUpdateInfo($updateId, false, true);
     }
 
     if(!$lang) {
@@ -56,6 +56,7 @@ function uupListEditions($lang = 'en-us', $updateId = 0) {
     $editionListFancy = array();
     foreach(array_keys($genPack[$lang]) as $edition) {
         if($edition == 'LXP') continue;
+        if($edition == 'FOD') continue;
 
         if(isset($fancyEditionNames[$edition])) {
             $fancyName = $fancyEditionNames[$edition];

listid.php

@@ -16,22 +16,31 @@ limitations under the License.
 */
 
 require_once dirname(__FILE__).'/shared/main.php';
+require_once dirname(__FILE__).'/shared/cache.php';
+require_once dirname(__FILE__).'/shared/fileinfo.php';
 
-function uupListIds($search = null, $sortByDate = 0) {
-    uupApiPrintBrand();
+function uupApiPrivateInvalidateFileinfoCache() {
+    $cache1 = new UupDumpCache('listid-0', false);
+    $cache2 = new UupDumpCache('listid-1', false);
 
-    if(!file_exists('fileinfo')) return array('error' => 'NO_FILEINFO_DIR');
+    $cache1->delete();
+    $cache2->delete();
+}
 
-    $files = scandir('fileinfo');
+function uupApiPrivateGetFromFileinfo($sortByDate = 0) {
+    $dirs = uupApiGetFileinfoDirs();
+    $fileinfo = $dirs['fileinfoData'];
+    $fileinfoRoot = $dirs['fileinfo'];
+
+    $files = scandir($fileinfo);
     $files = preg_grep('/\.json$/', $files);
 
     consoleLogger('Parsing database info...');
 
-    $cacheFile = 'cache/fileinfo_v2.json';
+    $cacheFile = $fileinfoRoot.'/cache.json';
     $cacheV2Version = 1;
 
-    $database = @file_get_contents($cacheFile);
-    $database = json_decode($database, true);
+    $database = uupApiReadJson($cacheFile);
 
     if(isset($database['version'])) {
         $version = $database['version'];
@@ -50,12 +59,13 @@ function uupListIds($search = null, $sortByDate = 0) {
     $newDb = array();
     $builds = array();
     foreach($files as $file) {
-        if($file == '.' || $file == '..') continue;
+        if($file == '.' || $file == '..')
+            continue;
+
         $uuid = preg_replace('/\.json$/', '', $file);
 
         if(!isset($database[$uuid])) {
-            $info = @file_get_contents('fileinfo/'.$file);
-            $info = json_decode($info, true);
+            $info = uupApiReadFileinfoMeta($uuid);
 
             $title = isset($info['title']) ? $info['title'] : 'UNKNOWN';
             $build = isset($info['build']) ? $info['build'] : 'UNKNOWN';
@@ -105,12 +115,7 @@ function uupListIds($search = null, $sortByDate = 0) {
         $builds[$tmp.$arch.$title.$uuid] = $temp;
     }
 
-    if(empty($buildAssoc)) {
-        return array(
-            'apiVersion' => uupApiVersion(),
-            'builds' => array(),
-        );
-    }
+    if(empty($buildAssoc)) return [];
 
     krsort($buildAssoc);
     $buildsNew = array();
@@ -141,7 +146,27 @@ function uupListIds($search = null, $sortByDate = 0) {
         if(!$success) consoleLogger('Failed to update database cache.');
     }
 
-    if($search) {
+    return $builds;
+}
+
+function uupListIds($search = null, $sortByDate = 0) {
+    uupApiPrintBrand();
+
+    $sortByDate = $sortByDate ? 1 : 0;
+
+    $res = "listid-$sortByDate";
+    $cache = new UupDumpCache($res, false);
+    $builds = $cache->get();
+    $cached = ($builds !== false);
+
+    if(!$cached) {
+        $builds = uupApiPrivateGetFromFileinfo($sortByDate);
+        if($builds === false) return ['error' => 'NO_FILEINFO_DIR'];
+
+        $cache->put($builds, 60);
+    }
+
+    if(count($builds) && $search != null) {
         if(!preg_match('/^regex:/', $search)) {
             $searchSafe = preg_quote($search, '/');
 

listlangs.php

@@ -1,6 +1,6 @@
 <?php
 /*
-Copyright 2019 UUP dump API authors
+Copyright 2022 UUP dump API authors
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -21,7 +21,7 @@ require_once dirname(__FILE__).'/updateinfo.php';
 
 function uupListLangs($updateId = 0) {
     if($updateId) {
-        $info = uupUpdateInfo($updateId);
+        $info = uupUpdateInfo($updateId, false, true);
     }
 
     if(isset($info['info'])) {
@@ -50,6 +50,9 @@ function uupListLangs($updateId = 0) {
         if(!count(array_diff(array_keys($val), array('LXP')))) {
             continue;
         }
+        if(!count(array_diff(array_keys($val), array('FOD')))) {
+            continue;
+        }
 
         if(isset($fancyLangNames[$key])) {
             $fancyName = $fancyLangNames[$key];

readme.md

@@ -10,7 +10,7 @@ Parameters:
    - **Supported values:** `amd64`, `x86`, `arm64`, `all`
 
  - `ring` - Channel to use when fetching information (Previously called Ring)
-   - **Supported values:** `Dev`, `Beta`, `ReleasePreview`, `Retail`
+   - **Supported values:** `Canary`, `Dev`, `Beta`, `ReleasePreview`, `Retail`
    - **Supported Ring values :** `WIF`, `WIS`, `RP`
 
  - `flight` - Content type to use when fetching information (Previously called Flight)
@@ -82,7 +82,7 @@ Parameters:
    - **Supported values:** any update UUID
 
 
-#### updateinfo.php: `uupUpdateInfo($updateId, $onlyInfo);`
+#### updateinfo.php: `uupUpdateInfo($updateId, $onlyInfo, $ignoreFiles);`
 Outputs specified information of specified `updateId`.
 
 Parameters:
@@ -92,12 +92,36 @@ Parameters:
  - `onlyInfo` - Key to output
    - **Supported values:** any string
 
+ - `ignoreFiles` - Skips the `files` key in the output
+   - **Supported values:** `true` or `false`
+
 #### shared/main.php: `uupApiVersion();`
 Returns version of the API.
 
 Parameters:
  - None
 
+#### shared/utils.php: `uupApiCheckUpdateId($updateId);`
+Checks if the provided update ID is correctly formatted.
+
+Parameters:
+ - `updateId` - update ID to check
+   - **Supported values:** Any string
+
+#### shared/utils.php: `uupApiIsServer($skuId);`
+Checks if the provided SKU ID is a Windows Sever SKU.
+
+Parameters:
+ - `skuId` - SKU ID to check
+   - **Supported values:** Any integer
+
+#### shared/utils.php: `uupApiBuildMajor($build);`
+Returns a build major of the build number.
+
+Parameters:
+ - `build` - Build number (for example 22621.1) to split
+   - **Supported values:** Any correctly formatted build number
+
 ### Error codes thrown by API
 **fetchupd.php**
  - UNKNOWN_ARCH
@@ -108,6 +132,7 @@ Parameters:
  - ILLEGAL_MINOR
  - NO_UPDATE_FOUND
  - EMPTY_FILELIST
+ - WU_REQUEST_FAILED
 
 **get.php**
  - UNSUPPORTED_LANG
@@ -118,6 +143,7 @@ Parameters:
  - MISSING_FILES
  - NO_FILES
  - XML_PARSE_ERROR
+ - WU_REQUEST_FAILED
 
 **listeditions.php**
  - UNSUPPORTED_LANG

shared/auths.php

@@ -25,19 +25,43 @@ function uupDevice() {
     return base64_encode(chunk_split($data, 1, "\0"));
 }
 
+function uupSaveCookieFromResponse($out) {
+    $outDecoded = html_entity_decode($out);
+    preg_match('/<NewCookie>.*?<\/NewCookie>|<GetCookieResult>.*?<\/GetCookieResult>/', $outDecoded, $cookieData);
+
+    if(empty($cookieData))
+        return false;
+
+    preg_match('/<Expiration>.*<\/Expiration>/', $cookieData[0], $expirationDate);
+    preg_match('/<EncryptedData>.*<\/EncryptedData>/', $cookieData[0], $encryptedData);
+
+    $expirationDate = preg_replace('/<Expiration>|<\/Expiration>/', '', $expirationDate[0]);
+    $encryptedData = preg_replace('/<EncryptedData>|<\/EncryptedData>/', '', $encryptedData[0]);
+
+    $cookieData = array(
+        'expirationDate' => $expirationDate,
+        'encryptedData' => $encryptedData,
+    );
+
+    $cookieStorage = new UupDumpCache('WuRequestCookie', false);
+    $cookieStorage->put($cookieData, false);
+
+    return $cookieData;
+}
+
+function uupInvalidateCookie() {
+    $cookieStorage = new UupDumpCache('WuRequestCookie', false);
+    $cookieInfo = $cookieStorage->delete();
+}
+
 function uupEncryptedData() {
-    $cookieInfo = @file_get_contents(dirname(__FILE__).'/cookie.json');
-    $cookieInfo = json_decode($cookieInfo, 1);
+    $cookieStorage = new UupDumpCache('WuRequestCookie', false);
+    $cookieInfo = $cookieStorage->get();
 
     if(empty($cookieInfo)) {
-        $postData = composeGetCookieRequest(uupDevice());
-        sendWuPostRequest('https://fe3.delivery.mp.microsoft.com/ClientWebService/client.asmx', $postData);
-
-        $encData = uupEncryptedData();
-    } else {
-        $encData = $cookieInfo['encryptedData'];
+        $data = sendWuPostRequestHelper('client', 'composeGetCookieRequest', [], false);
+        $cookieInfo = uupSaveCookieFromResponse($data['out']);
     }
 
-    return $encData;
+    return $cookieInfo['encryptedData'];
 }
-?>

shared/cache.php

@@ -0,0 +1,76 @@
+<?php
+/*
+Copyright 2022 eraseyourknees
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+class UupDumpCache {
+    private $cacheFile;
+    private $newCacheVersion = 1;
+
+    public function __construct($resource, private $isCompressed = true) {
+        $res = $resource."+cache_v".$this->newCacheVersion;
+        $cacheHash = hash('sha256', strtolower($res));
+        $ext = $isCompressed ? '.json.gz' : '.json';
+        $this->cacheFile = 'cache/'.$cacheHash.$ext;
+    }
+
+    public function getFileName() {
+        return $this->cacheFile;
+    }
+
+    public function delete() {
+        @unlink($this->cacheFile);
+    }
+
+    public function get() {
+        $cacheFile = $this->cacheFile;
+
+        if(!file_exists($cacheFile)) {
+            return false;
+        }
+
+        $cache = @file_get_contents($cacheFile);
+        if($this->isCompressed) $cache = @gzdecode($cache);
+
+        $cache = json_decode($cache, 1);
+
+        $expires = $cache['expires'];
+        $isExpired = ($expires !== false) && (time() > $expires);
+
+        if(empty($cache['content']) || $isExpired) {
+            $this->delete();
+            return false;
+        }
+
+        return $cache['content'];
+    }
+
+    public function put($content, $validity) {
+        $cacheFile = $this->cacheFile;
+        $expires = $validity ? time() + $validity : false;
+
+        $cache = array(
+            'expires' => $expires,
+            'content' => $content,
+        );
+    
+        if(!file_exists('cache')) mkdir('cache');
+
+        $cacheContent = json_encode($cache)."\n";
+        if($this->isCompressed) $cacheContent = @gzencode($cacheContent);
+
+        @file_put_contents($cacheFile, $cacheContent);
+    }
+}

shared/fileinfo.php

@@ -0,0 +1,94 @@
+<?php
+/*
+Copyright 2022 UUP dump API authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+require_once dirname(__FILE__).'/utils.php';
+
+function uupApiGetFileinfoDirs() {
+    $dirs = [];
+    $dirs['fileinfo'] = 'fileinfo';
+    $dirs['fileinfoMeta'] = $dirs['fileinfo'].'/metadata';
+    $dirs['fileinfoData'] = $dirs['fileinfo'].'/full';
+
+    foreach($dirs as $dir) {
+        if(!file_exists($dir)) mkdir($dir);
+    }
+
+    return $dirs;
+}
+
+function uupApiGetFileinfoName($updateId, $meta = false) {
+    $fileName = $updateId.'.json';
+    $dirs = uupApiGetFileinfoDirs();
+
+    $fileinfoMeta = $dirs['fileinfoMeta'].'/'.$fileName;
+    $fileinfoData = $dirs['fileinfoData'].'/'.$fileName;
+
+    return $meta ? $fileinfoMeta : $fileinfoData;
+}
+
+function uupApiFileInfoExists($updateId) {
+    return file_exists(uupApiGetFileinfoName($updateId));
+}
+
+function uupApiWriteFileinfoMeta($updateId, $info) {
+    if(isset($info['files']))
+        unset($info['files']);
+
+    $file = uupApiGetFileinfoName($updateId, true);
+    return uupApiWriteJson($file, $info);
+}
+
+function uupApiWriteFileinfo($updateId, $info) {
+    $file = uupApiGetFileinfoName($updateId);
+
+    if(uupApiWriteJson($file, $info) === false)
+        return false;
+
+    return uupApiWriteFileinfoMeta($updateId, $info);
+}
+
+function uupApiReadFileinfoMeta($updateId) {
+    $file = uupApiGetFileinfoName($updateId, true);
+
+    if(file_exists($file))
+        return uupApiReadJson($file);
+
+    $info = uupApiReadFileinfo($updateId, false);
+    if($info === false)
+        return false;
+
+    if(isset($info['files']))
+        unset($info['files']);
+
+    if(uupApiWriteFileinfoMeta($updateId, $info) === false)
+        return false;
+
+    return $info;
+}
+
+function uupApiReadFileinfo($updateId, $meta = false) {
+    if(!uupApiFileInfoExists($updateId))
+        return false;
+
+    if($meta === true)
+        return uupApiReadFileinfoMeta($updateId);
+
+    $file = uupApiGetFileinfoName($updateId);
+    $info = uupApiReadJson($file);
+
+    return $info;
+}

shared/main.php

@@ -16,7 +16,7 @@ limitations under the License.
 */
 
 function uupApiVersion() {
-    return '1.32.0';
+    return '1.41.0';
 }
 
 require_once dirname(__FILE__).'/auths.php';

shared/packs.php

@@ -1,6 +1,6 @@
 <?php
 /*
-Copyright 2021 UUP dump API authors
+Copyright 2022 UUP dump API authors
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -19,6 +19,7 @@ require_once dirname(__FILE__).'/../listid.php';
 
 function uupGetInfoTexts() {
     $fancyLangNames = array(
+        'neutral' => 'Any Language',
         'ar-sa' => 'Arabic (Saudi Arabia)',
         'bg-bg' => 'Bulgarian',
         'cs-cz' => 'Czech',
@@ -61,6 +62,9 @@ function uupGetInfoTexts() {
     );
 
     $fancyEditionNames = array(
+        'APP' => 'Microsoft Store Inbox Apps',
+        'APP_MOMENT' => 'Microsoft Store Moment Apps',
+        'FOD' => 'Features on Demand (Capabilities)',
         'CLOUD' => 'Windows S',
         'CLOUDN' => 'Windows S N',
         'CLOUDE' => 'Windows Lean',
@@ -199,165 +203,12 @@ function uupGetInfoTexts() {
 }
 
 function uupGetGenPacks($build = 15063, $arch = null, $updateId = null) {
-    $internalPacks = dirname(__FILE__).'/packs';
+    if(empty($updateId)) return [];
+    if(!file_exists('packs/'.$updateId.'.json.gz')) return [];
 
-    if(!file_exists($internalPacks.'/metadata.json')) {
-        if(!uupCreateInternalPacksMetadata($internalPacks)) {
-            return array();
-        }
-    }
-
-    if(!empty($updateId)) {
-        if(file_exists('packs/'.$updateId.'.json.gz')) {
-            $genPack = @gzdecode(@file_get_contents('packs/'.$updateId.'.json.gz'));
-            if(empty($genPack)) return array();
-
-            $genPack = json_decode($genPack, 1);
-            return $genPack;
-        }
-    }
-
-    $metadata = @file_get_contents($internalPacks.'/metadata.json');
-    if(empty($metadata)) {
-        return array();
-    } else {
-        $metadata = json_decode($metadata, 1);
-    }
-
-    $hashDetermined = 0;
-    $useAllHashesForBuild = 0;
-
-    if($updateId) {
-        if(isset($metadata['knownIds'][$updateId])) {
-            $hash = $metadata['knownIds'][$updateId];
-            $hashDetermined = 1;
-        }
-    }
-
-    if(!$hashDetermined) {
-        foreach($metadata['knownBuilds'] as $buildNum => $val) {
-            if($build < $buildNum) continue;
-            $useBuild = $buildNum;
-            break;
-        }
-
-        if(!isset($useBuild)) {
-            return array();
-        }
-
-        if(!$arch && !isset($metadata['knownBuilds'][$useBuild][$arch])) {
-            $genPack = array();
-            foreach($metadata['knownBuilds'][$useBuild] as $hash) {
-                $temp = @gzdecode(@file_get_contents($internalPacks.'/'.$hash.'.json.gz'));
-                if(!empty($temp)) {
-                    $temp = json_decode($temp, 1);
-                    $genPack = array_merge_recursive($genPack, $temp);
-                    unset($temp);
-                }
-            }
-        } elseif(!isset($metadata['knownBuilds'][$useBuild][$arch])) {
-            return array();
-        } else {
-            $hash = $metadata['knownBuilds'][$useBuild][$arch];
-        }
-    }
-
-    if(!isset($genPack)) {
-        $genPack = @gzdecode(@file_get_contents($internalPacks.'/'.$hash.'.json.gz'));
-        if(!empty($genPack)) {
-            $genPack = json_decode($genPack, 1);
-        } else {
-            $genPack = array();
-        }
-    }
+    $genPack = @gzdecode(@file_get_contents('packs/'.$updateId.'.json.gz'));
+    if(empty($genPack)) return [];
 
+    $genPack = json_decode($genPack, 1);
     return $genPack;
 }
-
-//Function to regenerate internal packs. Should not be used when not needed.
-function uupCreateInternalPacksMetadata($internalPacks) {
-    $metadataCreationAllowed = 0;
-    if(!$metadataCreationAllowed) return false;
-
-    $builds = uupListIds();
-    if(isset($ids['error'])) {
-        return false;
-    }
-
-    $builds = $builds['builds'];
-
-    if(!file_exists('packs')) return false;
-
-    if(!file_exists($internalPacks)) {
-        if(!mkdir($internalPacks)) {
-            return false;
-        }
-    } else {
-        rmdir($internalPacks);
-        mkdir($internalPacks);
-    }
-
-    $files = scandir('packs');
-    $files = preg_grep('/\.json.gz$/', $files);
-
-    $packs = array();
-    foreach($builds as $build) {
-        $uuid = $build['uuid'];
-        $file = $uuid.'.json.gz';
-
-        if(!file_exists('packs/'.$file)) continue;
-
-        $genPack = @gzdecode(@file_get_contents('packs/'.$file));
-        $hash = hash('sha1', $genPack);
-
-        if(!file_exists($internalPacks.'/'.$hash.'.json.gz')) {
-            if(!copy('packs/'.$file, $internalPacks.'/'.$hash.'.json.gz')) {
-                return false;
-            }
-        }
-
-        $packs['knownIds'][$uuid] = $hash;
-
-        $buildNum = explode('.', $build['build']);
-        $buildNum = $buildNum[0];
-
-        $packs['knownBuilds'][$buildNum][$build['arch']] = $hash;
-    }
-
-    file_put_contents($internalPacks.'/metadata.json', json_encode($packs)."\n");
-    return true;
-}
-
-//Emulation of legacy packs. Do not use in new scripts due to extremely slow process.
-function uupGetPacks($build = 15063) {
-    $returnArray = uupGetInfoTexts();
-    $genPack = uupGetGenPacks($build);
-
-    foreach($genPack as $lang => $editions) {
-        $packsForLangs[$lang] = array_keys($editions);
-        $packsForLangs[$lang][] = $lang;
-
-        foreach(array_keys($editions) as $edition) {
-            foreach($editions[$edition] as $name) {
-                $newName = preg_replace('/^cabs_|^metadataesd_|~31bf3856ad364e35/i', '', $name);
-                $newName = preg_replace('/~~\.|~\./', '.', $newName);
-                $newName = preg_replace('/~/', '-', $newName);
-                $newName = strtolower($newName);
-                $packs[$lang][$edition][] = $newName;
-            }
-
-            $editionPacks[$edition] = $edition;
-            $packs[$edition][$edition] = array();
-            $skipNeutral[$edition] = 1;
-            $skipLangPack[$edition] = 1;
-        }
-    }
-
-    $returnArray['packs'] = $packs;
-    $returnArray['packsForLangs'] = $packsForLangs;
-    $returnArray['editionPacks'] = $editionPacks;
-    $returnArray['skipNeutral'] = $skipNeutral;
-    $returnArray['skipLangPack'] = $skipLangPack;
-
-    return $returnArray;
-}