Merge branch 'feature/idf_tools_installer_ci' into 'master'

Build Windows tools installer in CI

Closes IDF-201

See merge request espressif/esp-idf!5583

tools/ci/config/build.yml

@@ -282,3 +282,82 @@ build_docker:
     # The workaround mentioned there works, but leaves around directories which need to be cleaned up manually.
     # Therefore, build a copy of the example located inside the container.
     - docker run --rm --workdir /opt/esp/idf/examples/get-started/blink ${DOCKER_TMP_IMAGE_NAME} idf.py build
+
+build_idf_exe:
+  stage: build
+  image: $CI_DOCKER_REGISTRY/esp32-toolchain-win-cross
+  tags:
+    - build
+  only:
+    refs:
+      - master
+      - /^release\/v/
+      - /^v\d+\.\d+(\.\d+)?($|-)/
+      - schedules
+  before_script: []
+  artifacts:
+    paths:
+      - tools/windows/idf_exe/build/idf-exe-v*.zip
+    expire_in: 3 days
+  script:
+    - cd tools/windows/idf_exe/
+    - mkdir build
+    - cd build
+    - cmake -DCMAKE_TOOLCHAIN_FILE=../toolchain-i686-w64-mingw32.cmake -DCMAKE_BUILD_TYPE=Release ..
+    - cmake --build .
+
+build_cmdlinerunner:
+  stage: build
+  image: $CI_DOCKER_REGISTRY/esp32-toolchain-win-cross
+  tags:
+    - build
+  only:
+    refs:
+      - master
+      - /^release\/v/
+      - /^v\d+\.\d+(\.\d+)?($|-)/
+      - schedules
+  before_script: []
+  artifacts:
+    paths:
+      - tools/windows/tool_setup/cmdlinerunner/build/cmdlinerunner.dll
+    expire_in: 3 days
+  script:
+    - cd tools/windows/tool_setup/cmdlinerunner
+    - mkdir build
+    - cd build
+    - cmake -DCMAKE_TOOLCHAIN_FILE=../toolchain-i686-w64-mingw32.cmake -DCMAKE_BUILD_TYPE=Release ..
+    - cmake --build .
+
+build_installer:
+  # using a different stage here to be able to use artifacts from build_cmdlinerunner job
+  stage: host_test
+  image: $CI_DOCKER_REGISTRY/wine-innosetup:1
+  tags:
+    - build
+  only:
+    refs:
+      - master
+      - /^release\/v/
+      - /^v\d+\.\d+(\.\d+)?($|-)/
+      - schedules
+  dependencies:
+    - build_cmdlinerunner
+  before_script: []
+  script:
+    - mkdir idf_tools_tmp
+    - export IDF_TOOLS_PATH=$PWD/idf_tools_tmp
+    - tools/idf_tools.py --non-interactive download --platform Windows-x86_64 all
+    - tools/idf_tools.py --tools-json tools/windows/tool_setup/tools_fallback.json --non-interactive download --platform Windows-x86_64 all
+    - mkdir tools/windows/tool_setup/dist
+    - mv idf_tools_tmp/dist/* tools/windows/tool_setup/dist/
+
+    - cd tools/windows/tool_setup/
+    - mkdir unzip
+    - cd unzip
+    - wget --no-verbose https://www.7-zip.org/a/7z1900-extra.7z
+    - 7zr e -y 7z1900-extra.7z
+    - cd ..
+
+    - wget --no-verbose https://dl.espressif.com/dl/esp-idf/idf_versions.txt
+    - iscc idf_tool_setup.iss

tools/idf_tools.py

@@ -53,6 +53,7 @@ import zipfile
 import errno
 import shutil
 import functools
+import copy
 from collections import OrderedDict, namedtuple
 
 try:
@@ -372,12 +373,19 @@ class IDFTool(object):
         self.options = IDFToolOptions(version_cmd, version_regex, version_regex_replace,
                                       [], OrderedDict(), install, info_url, license, strip_container_dirs)
         self.platform_overrides = []
+        self._platform = CURRENT_PLATFORM
         self._update_current_options()
 
+    def copy_for_platform(self, platform):
+        result = copy.deepcopy(self)
+        result._platform = platform
+        result._update_current_options()
+        return result
+
     def _update_current_options(self):
         self._current_options = IDFToolOptions(*self.options)
         for override in self.platform_overrides:
-            if CURRENT_PLATFORM not in override['platforms']:
+            if self._platform not in override['platforms']:
                 continue
             override_dict = override.copy()
             del override_dict['platforms']
@@ -422,6 +430,8 @@ class IDFTool(object):
         Returns 'unknown' if tool returns something from which version string
         can not be extracted.
         """
+        # this function can not be called for a different platform
+        assert self._platform == CURRENT_PLATFORM
         cmd = self._current_options.version_cmd
         try:
             version_cmd_result = run_cmd_check_output(cmd, None, extra_paths)
@@ -447,7 +457,7 @@ class IDFTool(object):
     def get_recommended_version(self):
         recommended_versions = [k for k, v in self.versions.items()
                                 if v.status == IDFToolVersion.STATUS_RECOMMENDED
-                                and v.compatible_with_platform()]
+                                and v.compatible_with_platform(self._platform)]
         assert len(recommended_versions) <= 1
         if recommended_versions:
             return recommended_versions[0]
@@ -456,7 +466,7 @@ class IDFTool(object):
     def get_preferred_installed_version(self):
         recommended_versions = [k for k in self.versions_installed
                                 if self.versions[k].status == IDFToolVersion.STATUS_RECOMMENDED
-                                and self.versions[k].compatible_with_platform()]
+                                and self.versions[k].compatible_with_platform(self._platform)]
         assert len(recommended_versions) <= 1
         if recommended_versions:
             return recommended_versions[0]
@@ -467,6 +477,8 @@ class IDFTool(object):
         Checks whether the tool can be found in PATH and in global_idf_tools_path.
         Writes results to self.version_in_path and self.versions_installed.
         """
+        # this function can not be called for a different platform
+        assert self._platform == CURRENT_PLATFORM
         # First check if the tool is in system PATH
         try:
             ver_str = self.check_version()
@@ -504,9 +516,9 @@ class IDFTool(object):
 
     def download(self, version):
         assert(version in self.versions)
-        download_obj = self.versions[version].get_download_for_platform(PYTHON_PLATFORM)
+        download_obj = self.versions[version].get_download_for_platform(self._platform)
         if not download_obj:
-            fatal('No packages for tool {} platform {}!'.format(self.name, PYTHON_PLATFORM))
+            fatal('No packages for tool {} platform {}!'.format(self.name, self._platform))
             raise DownloadError()
 
         url = download_obj.url
@@ -543,7 +555,7 @@ class IDFTool(object):
         # Currently this is called after calling 'download' method, so here are a few asserts
         # for the conditions which should be true once that method is done.
         assert (version in self.versions)
-        download_obj = self.versions[version].get_download_for_platform(PYTHON_PLATFORM)
+        download_obj = self.versions[version].get_download_for_platform(self._platform)
         assert (download_obj is not None)
         archive_name = os.path.basename(download_obj.url)
         archive_path = os.path.join(global_idf_tools_path, 'dist', archive_name)
@@ -985,7 +997,7 @@ def action_export(args):
         raise SystemExit(1)
 
 
-def apply_mirror_prefix_map(args, tool_obj, tool_version):
+def apply_mirror_prefix_map(args, tool_download_obj):
     """Rewrite URL for given tool_obj, given tool_version, and current platform,
        if --mirror-prefix-map flag or IDF_MIRROR_PREFIX_MAP environment variable is given.
     """
@@ -998,21 +1010,62 @@ def apply_mirror_prefix_map(args, tool_obj, tool_version):
             warn('Both IDF_MIRROR_PREFIX_MAP environment variable and --mirror-prefix-map flag are specified, ' +
                  'will use the value from the command line.')
         mirror_prefix_map = args.mirror_prefix_map
-    download_obj = tool_obj.versions[tool_version].get_download_for_platform(PYTHON_PLATFORM)
+    if mirror_prefix_map and tool_download_obj:
-    if mirror_prefix_map and download_obj:
         for item in mirror_prefix_map:
             if URL_PREFIX_MAP_SEPARATOR not in item:
                 warn('invalid mirror-prefix-map item (missing \'{}\') {}'.format(URL_PREFIX_MAP_SEPARATOR, item))
                 continue
             search, replace = item.split(URL_PREFIX_MAP_SEPARATOR, 1)
-            old_url = download_obj.url
+            old_url = tool_download_obj.url
             new_url = re.sub(search, replace, old_url)
             if new_url != old_url:
                 info('Changed download URL: {} => {}'.format(old_url, new_url))
-                download_obj.url = new_url
+                tool_download_obj.url = new_url
                 break
 
 
+def action_download(args):
+    tools_info = load_tools_info()
+    tools_spec = args.tools
+
+    if args.platform not in PLATFORM_FROM_NAME:
+        fatal('unknown platform: {}' % args.platform)
+        raise SystemExit(1)
+    platform = PLATFORM_FROM_NAME[args.platform]
+
+    tools_info_for_platform = OrderedDict()
+    for name, tool_obj in tools_info.items():
+        tool_for_platform = tool_obj.copy_for_platform(platform)
+        tools_info_for_platform[name] = tool_for_platform
+
+    if 'all' in tools_spec:
+        tools_spec = [k for k, v in tools_info_for_platform.items() if v.get_install_type() != IDFTool.INSTALL_NEVER]
+        info('Downloading tools for {}: {}'.format(platform, ', '.join(tools_spec)))
+
+    for tool_spec in tools_spec:
+        if '@' not in tool_spec:
+            tool_name = tool_spec
+            tool_version = None
+        else:
+            tool_name, tool_version = tool_spec.split('@', 1)
+        if tool_name not in tools_info_for_platform:
+            fatal('unknown tool name: {}'.format(tool_name))
+            raise SystemExit(1)
+        tool_obj = tools_info_for_platform[tool_name]
+        if tool_version is not None and tool_version not in tool_obj.versions:
+            fatal('unknown version for tool {}: {}'.format(tool_name, tool_version))
+            raise SystemExit(1)
+        if tool_version is None:
+            tool_version = tool_obj.get_recommended_version()
+        assert tool_version is not None
+        tool_spec = '{}@{}'.format(tool_name, tool_version)
+
+        info('Downloading {}'.format(tool_spec))
+        apply_mirror_prefix_map(args, tool_obj.versions[tool_version].get_download_for_platform(platform))
+
+        tool_obj.download(tool_version)
+
+
 def action_install(args):
     tools_info = load_tools_info()
     tools_spec = args.tools
@@ -1049,7 +1102,7 @@ def action_install(args):
             continue
 
         info('Installing {}'.format(tool_spec))
-        apply_mirror_prefix_map(args, tool_obj, tool_version)
+        apply_mirror_prefix_map(args, tool_obj.versions[tool_version].get_download_for_platform(PYTHON_PLATFORM))
 
         tool_obj.download(tool_version)
         tool_obj.install(tool_version)
@@ -1174,14 +1227,22 @@ def main(argv):
                                                 'will be used instead. If this flag is given, the version in PATH ' +
                                                 'will be used.', action='store_true')
     install = subparsers.add_parser('install', help='Download and install tools into the tools directory')
-    if IDF_MAINTAINER:
-        install.add_argument('--mirror-prefix-map', nargs='*',
-                             help='Pattern to rewrite download URLs, with source and replacement separated by comma.' +
-                                  ' E.g. http://foo.com,http://test.foo.com')
     install.add_argument('tools', nargs='*', help='Tools to install. ' +
                                                   'To install a specific version use tool_name@version syntax.' +
                                                   'Use \'all\' to install all tools, including the optional ones.')
 
+    download = subparsers.add_parser('download', help='Download the tools into the dist directory')
+    download.add_argument('--platform', help='Platform to download the tools for')
+    download.add_argument('tools', nargs='+', help='Tools to download. ' +
+                                                   'To download a specific version use tool_name@version syntax.' +
+                                                   'Use \'all\' to download all tools, including the optional ones.')
+
+    if IDF_MAINTAINER:
+        for subparser in [download, install]:
+            subparser.add_argument('--mirror-prefix-map', nargs='*',
+                                   help='Pattern to rewrite download URLs, with source and replacement separated by comma.' +
+                                        ' E.g. http://foo.com,http://test.foo.com')
+
     install_python_env = subparsers.add_parser('install-python-env',
                                                help='Create Python virtual environment and install the ' +
                                                     'required Python packages')