---
- name: Reset checkout to clean state  # noqa command-instead-of-module
  ansible.builtin.shell:
    cmd: "git clean -dfx; git reset --hard"
    chdir: "{{ tensorflow_build_dir }}/"
  when: tensorflow_version == 'git'

- name: (Python {{ py_ver }}) fetch TensorFlow source archive
  when: tensorflow_version != 'git'
  block:
    - name: Remove TensorFlow{{ variant_name }} {{ tensorflow_version }} sources from previous build
      ansible.builtin.file:
        path: "{{ tensorflow_build_dir }}/"
        state: absent

    - name: Fetch TensorFlow
      ansible.builtin.include_role:
        name: tensorflow
        tasks_from: fetch.yml

- name: (Python {{ py_ver }}) patch TensorFlow tree
  ansible.builtin.include_role:
    name: tensorflow
    tasks_from: patch_tree.yml

- name: (Python {{ py_ver }}) copy upper constraints for hermetic builds
  local_action:
    module: ansible.builtin.template
    src: "hermetic-upper-constraints.txt"
    dest: "{{ build_dir }}/configs/ldcg-python-manylinux-tensorflow/upper-constraints.txt"
    mode: 0644  # needs to be readable in container as well
    force: true

- name: (Python {{ py_ver }}) build Python versioned container
  community.docker.docker_image:
    name: "multipython-py{{ py_ver }}-{{ image_prefix }}"
    build:
      path: "{{ build_dir }}/configs/ldcg-python-manylinux-tensorflow/"
      args:
        tag_prefix: "{{ image_prefix }}"
        py_major_minor_version: "{{ py_ver }}"
    source: build
    force_source: true

- name: (Python {{ py_ver }}) start build container
  community.docker.docker_container:
    name: "tf-py{{ py_ver }}"
    image: "multipython-py{{ py_ver }}-{{ image_prefix }}"
    state: started
    restart: true
    volumes:
      - "{{ tensorflow_build_dir }}:/tf/tensorflow"
      - "{{ build_dir }}/wheels:/tf/pkg"
      - "{{ build_dir }}/disk-cache:/tf/cache"
      - "{{ build_dir }}/bazel-cache:/tf/bazel-cache"
    detach: true
    user: buildslave
    working_dir: "/tf/tensorflow"
    network_mode: host
    command: "sleep infinity"
    timeout: 6000

- name: (Python {{ py_ver }}) create buildslave bazel cache dir
  community.docker.docker_container_exec:
    container: "tf-py{{ py_ver }}"
    command: |
      install -d -m0777 .cache/bazel/_bazel_buildslave
    chdir: "/home/buildslave/"

- name: (Python {{ py_ver }}) create link for bazel cache
  community.docker.docker_container_exec:
    container: "tf-py{{ py_ver }}"
    command: |
      ln -sf /tf/bazel-cache bazel/_bazel_buildslave/cache
    chdir: "/home/buildslave/.cache/"

- name: (Python {{ py_ver }}) create link for bazelisk cache
  community.docker.docker_container_exec:
    container: "tf-py{{ py_ver }}"
    command: |
      ln -sf /tf/bazel-cache bazelisk
    chdir: "/home/buildslave/.cache/"

- name: (Python {{ py_ver }}) create link for pip cache
  community.docker.docker_container_exec:
    container: "tf-py{{ py_ver }}"
    command: |
      ln -sf /tf/cache pip
    chdir: "/home/buildslave/.cache/"

- name: (Python {{ py_ver }}) collect list of installed Python packages for build
  community.docker.docker_container_exec:
    container: "tf-py{{ py_ver }}"
    command: "/usr/local/bin/python3 -m pip list -v"
    chdir: "/tf/tensorflow"
  register: pip_list_output

- name: Show list of installed Python packages for build
  ansible.builtin.debug:
    var: pip_list_output.stdout

- name: (Python {{ py_ver }}) build TensorFlow{{ variant_name }} {{ tensorflow_version }}
  community.docker.docker_container_exec:
    container: "tf-py{{ py_ver }}"
    command: >
      bazel --bazelrc=/usertools/{{ bazelrc_file }} build
            --config=sigbuild_local_cache
            {{ "--config=mkl_aarch64_threadpool" if onednn_build }}
            {{ "--test_env=TF_ENABLE_ONEDNN_OPTS=1" if onednn_enable_opt }}
            {{ "--repo_env=WHEEL_NAME=tensorflow_aarch64" if not tf_ver.value.old_pip_package }}
            -- //tensorflow/tools/pip_package:{{ "build_pip_package" if tf_ver.value.old_pip_package else "wheel" }}
    chdir: "/tf/tensorflow"

- name: (Python {{ py_ver }}) build Python package
  community.docker.docker_container_exec:
    container: "tf-py{{ py_ver }}"
    command: |
      ./bazel-bin/tensorflow/tools/pip_package/build_pip_package \
            --project_name tensorflow_aarch64 /tf/pkg
    chdir: "/tf/tensorflow"
  when: tf_ver.value.old_pip_package

- name: (Python {{ py_ver }}) Copy built wheel
  community.docker.docker_container_exec:
    container: "tf-py{{ py_ver }}"
    command: |
      find ./bazel-bin/tensorflow/tools/pip_package -iname '*.whl' -exec cp '{}' /tf/pkg ';'
    chdir: "/tf/tensorflow"
  when: not tf_ver.value.old_pip_package

- name: (Python {{ py_ver }}) Run TensorFlow testsuite
  when: tf_ver.value.run_unit_tests
  block:
    - name: (Python {{ py_ver }}) cleanup Bazel install directory
      community.docker.docker_container_exec:
        container: "tf-py{{ py_ver }}"
        command: "rm -rf bazel/_bazel_buildslave/install"
        chdir: "/home/buildslave/.cache/"

    - name: (Python {{ py_ver }}) create script to run tests
      ansible.builtin.template:
        src: "run-tests.sh.j2"
        dest: "{{ tensorflow_build_dir }}/run-tests.sh"
        mode: 0755
        force: true

    - name: (Python {{ py_ver }}) run TensorFlow{{ variant_name }} {{ tensorflow_version }} tests
      community.docker.docker_container_exec:
        container: "tf-py{{ py_ver }}"
        command: "bash run-tests.sh"
        chdir: "/tf/tensorflow"
      register: test_output
      ignore_errors: true

    - name: (Python {{ py_ver }}) collect list of installed Python packages
      community.docker.docker_container_exec:
        container: "tf-py{{ py_ver }}"
        command: "/usr/local/bin/python3 -m pip list -v"
        chdir: "/tf/tensorflow"
      register: pip_output

    - name: (Python {{ py_ver }}) save list of installed Python packages
      ansible.builtin.template:
        src: "pip-stdout.j2"
        dest: "{{ wheels_dir }}/log-pip-py{{ py_ver }}.txt"
        mode: 0444

    - name: (Python {{ py_ver }}) save tests output to file
      ansible.builtin.template:
        src: "log-stdout.j2"
        dest: "{{ wheels_dir }}/log-tests-py{{ py_ver }}.txt"
        mode: 0444

    - name: (Python {{ py_ver }}) save tests stderr output to file
      ansible.builtin.template:
        src: "log-stderr.j2"
        dest: "{{ wheels_dir }}/log-tests-py{{ py_ver }}-stderr.txt"
        mode: 0444

    - name: (Python {{ py_ver }}) Extract number of tests run
      ansible.builtin.set_fact:
        ut_tests_line: "{{ test_output.stdout | regex_search('^Executed [0-9]+ out of [0-9]+ tests: [0-9]+ tests pass.*', multiline=True) }}"

    - name: (Python {{ py_ver }}) Show number of tests run
      ansible.builtin.debug:
        var: ut_tests_line

    - name: (Python {{ py_ver }}) Extract failed tests
      ansible.builtin.set_fact:
        tests_failed: "{{ test_output.stderr | regex_findall('FAILED: .*', multiline=True) }}"
        # log-tests-py3.8-stderr.txt:FAILED: //tensorflow/tools/api/tests:api_compatibility_test (Summary)
        tests_flaky: "{{ test_output.stdout | regex_findall('.*FLAKY.*', multiline=True) }}"
        # log-tests-py3.9.txt://tensorflow/python/data/experimental/kernel_tests/service:worker_tags_test FLAKY, failed in 2 out of 34 in 900.3s

    - name: (Python {{ py_ver }}) Show which tests are flaky
      ansible.builtin.debug:
        var: tests_flaky

    - name: (Python {{ py_ver }}) Show which tests failed
      ansible.builtin.debug:
        var: tests_failed

- name: (Python {{ py_ver }}) Convert wheel to manylinux2014 compatible one
  community.docker.docker_container_exec:
    container: "tf-py{{ py_ver }}"
    command: "/usertools/rename_and_verify_wheels.sh"
    chdir: "/tf/tensorflow"

- name: (Python {{ py_ver }}) Get list of wheel files
  ansible.builtin.find:
    path: "{{ wheels_dir }}"
    patterns: "tensorflow*.whl"
    file_type: file
  register: wheel_files

- name: (Python {{ py_ver }}) Get filename of latest wheel
  ansible.builtin.set_fact:
    wheel_file: "{{ wheel_files.files | sort(attribute='mtime') | last }}"

- name: (Python {{ py_ver }}) move manylinux2014 wheel outside
  ansible.builtin.copy:
    src: "{{ wheel_file.path }}"
    dest: "{{ build_dir }}/ml_wheels/"
    mode: 0444

- name: (Python {{ py_ver }}) Remove manylinux2014 wheel
  ansible.builtin.file:
    path: "{{ wheel_file.path }}"
    state: absent

- name: (Python {{ py_ver }}) stop build container
  community.docker.docker_container:
    name: "tf-py{{ py_ver }}"
    image: "linaro/tensorflow-arm64-build:latest-python{{ py_ver }}"
    state: absent
    timeout: 6000
