sage_bootstrap.download.mirror_list: Use master list of download sour…

…ces in SAGE_ROOT/.upstream.d

.upstream.d/10-SAGE_SERVER

@@ -0,0 +1,2 @@
+# When SAGE_SERVER is set, it should be an https/https server in the format of Sage mirrors.
+${SAGE_SERVER}/spkg/upstream/${SPKG}/

.upstream.d/20-github.amrom.workers.dev-sagemath-sage-releases

@@ -0,0 +1,2 @@
+# Upstream packages as uploaded as GitHub release assets.
+https://github.com/sagemath/sage/releases/download/10.1/

.upstream.d/30-www.sagemath.org-mirror_list

@@ -0,0 +1 @@
+https://www.sagemath.org/mirror_list

build/sage_bootstrap/download/mirror_list.py

@@ -19,7 +19,7 @@
 log = logging.getLogger()
 
 from sage_bootstrap.compat import urllib, urlparse
-from sage_bootstrap.env import SAGE_DISTFILES
+from sage_bootstrap.env import SAGE_DISTFILES, SAGE_ROOT
 
 from fcntl import flock, LOCK_SH, LOCK_EX
 from errno import ENOLCK
@@ -41,16 +41,60 @@ class MirrorListException(RuntimeError):
     pass
 
 
-MIRRORLIST_FILENAME = os.path.join(SAGE_DISTFILES, 'mirror_list')
+class MirrorList(object):
 
+    def __init__(self):
+        self.sources = []
+        upstream_d = os.path.join(SAGE_ROOT, '.upstream.d')
+        for fname in sorted(os.listdir(upstream_d)):
+            if '~' in fname or '#' in fname:
+                # Ignore auto-save and backup files
+                continue
+            try:
+                with open(os.path.join(upstream_d, fname), 'r') as f:
+                    for line in f:
+                        line = line.replace('${SAGE_ROOT}', SAGE_ROOT)
+                        line = line.replace('${SAGE_DISTFILES}', SAGE_DISTFILES)
+                        if '${SAGE_SERVER}' in line:
+                            SAGE_SERVER = os.environ.get("SAGE_SERVER", "")
+                            if not SAGE_SERVER:
+                                continue
+                            line = line.replace('${SAGE_SERVER}',)
+                        line = line.strip()
+                        if line.startswith('#'):
+                            continue
+                        if not line:
+                            continue
+                        if line.endswith('mirror_list'):
+                            cache_filename = os.path.join(SAGE_DISTFILES, line.rpartition('/')[2])
+                            self.sources.append(MirrorList_from_url(line, cache_filename))
+                        else:
+                            self.sources.append([line])
+            except IOError:
+                # Silently ignore files that do not exist
+                pass
 
-class MirrorList(object):
+    def __iter__(self):
+        """
+        Iterate through the list of mirrors.
+
+        This is the main entry point into the mirror list. Every
+        script should just use this function to try mirrors in order
+        of preference. This will not just yield the official mirrors,
+        but also urls for packages that are currently being tested.
+        """
+        for source in self.sources:
+            for mirror in source:
+                yield mirror
+
+
+class MirrorList_from_url(object):
 
-    URL = 'http://www.sagemath.org/mirror_list'
     MAXAGE = 24*60*60   # seconds
 
-    def __init__(self):
-        self.filename = MIRRORLIST_FILENAME
+    def __init__(self, url, filename):
+        self.url = url
+        self.filename = filename
         self._mirrors = None
 
     @property