[PLAT-13554] move systemd setup to configure stage

Summary:
Previously, we were having systemd units configuration as part of the provisioning step.
Given the fact we no longer support centos7/al2 starting from 2024.1 we can remove the support for system level systemd & make user level one as the default.
This diff changes the systemd installation during configure phase as that won't need the sudo user & can be leveraged during onprem manual deployments.

Leaves the control script as is - as the YBA could be managing the older universes with system level systemd

Test Plan:
iTest pipeline

Created a universe with otelCollecterEnabled - Verified that the collector is running & the logs are pushed to configured provider.
Enabled collector on the deployed universe - Verified that the collector is running & the logs are pushed to configured provider.

Created an universe on centos with 2.19 DB build on master
Started the YBA with the above patch.
Added a node in the above universe.
Performed basic operation like gFlag upgrade.
Verified we are still using system level systemd.
Configured otel collector in the universe. Verified logs are exported to the configured provider.

Reviewers: muthu, anijhawan, sanketh, nsingh, amalyshev, sneelakantan

Reviewed By: muthu, amalyshev

Subscribers: skurapati, yugaware

Differential Revision: https://phorge.dev.yugabyte.com/D34323

managed/devops/opscli/ybops/cloud/common/method.py

@@ -1261,6 +1261,17 @@ def prepare(self):
                                  required=False,
                                  help="The maximum number of checking the clock skew before "
                                       "failing.")
+        self.parser.add_argument("--install_otel_collector", action="store_true")
+        self.parser.add_argument('--otel_col_config_file', default=None,
+                                 help="Path to OpenTelemetry Collector config file.")
+        self.parser.add_argument('--otel_col_aws_access_key', default=None,
+                                 help="AWS Access Key used for logs export")
+        self.parser.add_argument('--otel_col_aws_secret_key', default=None,
+                                 help="AWS Secret Key used for logs export.")
+        self.parser.add_argument('--otel_col_gcp_creds_file', default=None,
+                                 help="Path to GCP credentials file used for logs export.")
+        self.parser.add_argument('--ycql_audit_log_level', default=None,
+                                 help="YCQL audit log level.")
 
     def get_ssh_user(self):
         # Force the yugabyte user for configuring instances. The configure step performs YB specific
@@ -1468,12 +1479,13 @@ def callback(self, args):
                     logging.info("[app] Copying package {} to {} took {:.3f} sec".format(
                         ybc_package_path, args.search_pattern, time.time() - start_time))
 
+        if args.local_package_path:
+            self.extra_vars.update({"local_package_path": args.local_package_path})
+        else:
+            logging.warn("Local Directory Tarball Path not specified skipping")
+            return
+
         if args.install_third_party_packages:
-            if args.local_package_path:
-                self.extra_vars.update({"local_package_path": args.local_package_path})
-            else:
-                logging.warn("Local Directory Tarball Path not specified skipping")
-                return
             self.cloud.setup_ansible(args).run(
                 "install-third-party.yml", self.extra_vars, host_info)
             return
@@ -1570,6 +1582,19 @@ def callback(self, args):
                 args.encryption_key_source_file, args.encryption_key_target_dir))
             self.cloud.create_encryption_at_rest_file(self.extra_vars, self.extra_vars)
 
+        if args.install_otel_collector:
+            self.extra_vars.update({"install_otel_collector": args.install_otel_collector})
+        if args.otel_col_config_file:
+            self.extra_vars.update({"otel_col_config_file_local": args.otel_col_config_file})
+        if args.otel_col_aws_access_key:
+            self.extra_vars.update({"otel_col_aws_access_key": args.otel_col_aws_access_key})
+        if args.otel_col_aws_secret_key:
+            self.extra_vars.update({"otel_col_aws_secret_key": args.otel_col_aws_secret_key})
+        if args.otel_col_gcp_creds_file:
+            self.extra_vars.update({"otel_col_gcp_creds_local": args.otel_col_gcp_creds_file})
+        if args.ycql_audit_log_level:
+            self.extra_vars.update({"ycql_audit_log_level": args.ycql_audit_log_level})
+
         if args.reset_master_state and args.extra_gflags is not None:
             delete_paths = []
             extra_gflags = self.extra_vars.get('extra_gflags')
@@ -2117,6 +2142,7 @@ def add_extra_args(self):
         self.parser.add_argument("--local_package_path",
                                  required=False,
                                  help="Path to local directory with third party software tarballs.")
+        self.parser.add_argument("--use_sudo", action='store_true', default=False)
 
     def callback(self, args):
         host_info = self.cloud.get_host_info(args)
@@ -2146,6 +2172,8 @@ def callback(self, args):
             self.extra_vars.update({"otel_col_gcp_creds_local": args.otel_col_gcp_creds_file})
         if args.ycql_audit_log_level:
             self.extra_vars.update({"ycql_audit_log_level": args.ycql_audit_log_level})
+        if args.use_sudo:
+            self.extra_vars.update({"use_sudo": args.use_sudo})
 
         if wait_for_server(self.extra_vars):
             self.cloud.setup_ansible(args).run("yb-otel-collector.yml",

managed/devops/roles/configure-cluster-server/tasks/prepare-configure-server.yml

@@ -132,6 +132,160 @@
     - yb-prebuilt-ami
     - install-software
 
+- block:
+  # Configure the user level systemd for yb-services.
+  - name: Configure | Register yb user id
+    shell:
+      cmd: "id -u {{ user_name }}"
+    register: output
+
+  - name: Configure | Add yb user bash profile
+    lineinfile:
+      dest: "{{ yb_home_dir }}/.bash_profile"
+      line: ". {{ yb_home_dir }}/.bashrc"
+      owner: "{{ user_name }}"
+      group: "{{ user_name }}"
+      create: yes
+      state: present
+
+  - name: Configure | Export XDG_RUNTIME_DIR for yb user
+    lineinfile:
+      dest: "{{ yb_home_dir }}/.bashrc"
+      line: "export XDG_RUNTIME_DIR=/run/user/{{ output.stdout }}"
+      owner: "{{ user_name }}"
+      group: "{{ user_name }}"
+      create: yes
+      state: present
+
+  - name: Configure | Add user systemd directory
+    file:
+      path: "{{ yb_home_dir }}/.config/systemd/user/"
+      state: directory
+      owner: "{{ user_name }}"
+      group: "{{ user_name }}"
+      mode: 0744
+
+  - set_fact:
+      systemd_dir: "{{ yb_home_dir }}/.config/systemd/user"
+
+  - name: Configure | Enable linger for {{ user_name }} user
+    shell:
+        cmd: "loginctl enable-linger {{ user_name }}"
+
+  - name: Configure | setup-postgres-cgroups
+    include_role:
+      name: setup-cgroup
+
+  - name: Configure | Setup OpenTelemetry Collector
+    include_role:
+      name: manage_otel_collector
+    when: install_otel_collector is defined and install_otel_collector|bool
+
+  # Adding systemd service files for yb-master and yb-tserver
+  - name: Configure | Add yb-master.service
+    template:
+      src: yb-master.service
+      dest: "{{ systemd_dir }}/yb-master.service"
+      owner: "{{ user_name }}"
+      group: "{{ user_name }}"
+      mode: 0644
+
+  - name: Configure | Add yb-tserver.service
+    template:
+      src: yb-tserver.service
+      dest: "{{ systemd_dir }}/yb-tserver.service"
+      owner: "{{ user_name }}"
+      group: "{{ user_name }}"
+      mode: 0644
+
+  # Adding systemd service files for yb-controller
+  - name: Configure | Add yb-controller.service
+    template:
+      src: "yb-controller.service"
+      dest: "{{ systemd_dir }}/yb-controller.service"
+      owner: "{{ user_name }}"
+      group: "{{ user_name }}"
+      mode: 0644
+
+  # Adding yb-bind_check service files for the availability of the port
+  - name: Configure | Add yb-bind_check.service
+    template:
+      src: "yb-bind_check.service"
+      dest: "{{ systemd_dir }}/yb-bind_check.service"
+      owner: "{{ user_name }}"
+      group: "{{ user_name }}"
+      mode: 0644
+
+  # Adding systemd service and timer files for zip_purge_yb_logs
+  - name: Configure | Add yb-zip_purge_yb_logs.service
+    template:
+      src: yb-zip_purge_yb_logs.service
+      dest: "{{ systemd_dir }}/yb-zip_purge_yb_logs.service"
+      owner: "{{ user_name }}"
+      group: "{{ user_name }}"
+      mode: 0644
+
+  - name: Configure | Add yb-zip_purge_yb_logs.timer
+    template:
+      src: yb-zip_purge_yb_logs.timer
+      dest: "{{ systemd_dir }}/yb-zip_purge_yb_logs.timer"
+      owner: "{{ user_name }}"
+      group: "{{ user_name }}"
+      mode: 0644
+
+  # Adding systemd service and timer files for clean_cores
+  - name: Configure | Add yb-clean_cores.service
+    template:
+      src: yb-clean_cores.service
+      dest: "{{ systemd_dir }}/yb-clean_cores.service"
+      owner: "{{ user_name }}"
+      group: "{{ user_name }}"
+      mode: 0644
+
+  - name: Configure | Add yb-clean_cores.timer
+    template:
+      src: yb-clean_cores.timer
+      dest: "{{ systemd_dir }}/yb-clean_cores.timer"
+      owner: "{{ user_name }}"
+      group: "{{ user_name }}"
+      mode: 0644
+
+  # Adding systemd service and timer files for collect_metrics
+  - name: Configure | Add yb-collect_metrics.service
+    template:
+      src: yb-collect_metrics.service
+      dest: "{{ systemd_dir }}/yb-collect_metrics.service"
+      owner: "{{ user_name }}"
+      group: "{{ user_name }}"
+      mode: 0644
+
+  - name: Configure | Add yb-collect_metrics.timer
+    template:
+      src: yb-collect_metrics.timer
+      dest: "{{ systemd_dir }}/yb-collect_metrics.timer"
+      owner: "{{ user_name }}"
+      group: "{{ user_name }}"
+      mode: 0644
+
+  - name: Configure | Create or Update .bashrc file to display systemd mode upon ssh login
+    blockinfile:
+      dest: "{{ yb_home_dir }}/.bash_profile"
+      block: |
+        ORANGE='\033[0;33m'
+        NC='\033[0m' # No Color
+
+        if [[ $- == *i* ]]
+            then export PS1="[\u@\h ~]\$ "
+            echo -e "Use ${ORANGE}'systemctl --user'${NC} to manage YB software"
+        fi
+      create: yes
+      state: present
+      mode: 0644
+
+  tags: systemd_upgrade
+  when: (systemd_option and not ((ansible_os_family == 'RedHat' and ansible_distribution_major_version == '7')
+        or (ansible_distribution == 'Amazon' and ansible_distribution_major_version == '2')))
+
 - block:
   # Remove old cron jobs that clean cores, purge logs and collect metrics
   - name: Remove old cron job that cleans cores (for upgrades)

managed/devops/roles/provision-cluster-server/templates/yb-controller.service → managed/devops/roles/configure-cluster-server/templates/yb-controller.service

@@ -10,11 +10,11 @@ PathExists={{yb_home_dir}}/controller/bin/yb-controller-server
 PathExists={{yb_home_dir}}/controller/conf/server.conf
 
 [Service]
+# Start
 {% if ansible_os_family == 'RedHat' and (ansible_distribution_major_version == '7' or (ansible_distribution == 'Amazon' and ansible_distribution_major_version == '2')) %}
 User={{ user_name }}
 Group={{ user_name }}
 {% endif %}
-# Start
 ExecStart={{yb_home_dir}}/controller/bin/yb-controller-server \
     --flagfile {{yb_home_dir}}/controller/conf/server.conf
 Restart=always

managed/devops/roles/provision-cluster-server/templates/yb-master.service → managed/devops/roles/configure-cluster-server/templates/yb-master.service

@@ -10,11 +10,11 @@ PathExists={{yb_home_dir}}/master/bin/yb-master
 PathExists={{yb_home_dir}}/master/conf/server.conf
 
 [Service]
+# Start
 {% if ansible_os_family == 'RedHat' and (ansible_distribution_major_version == '7' or (ansible_distribution == 'Amazon' and ansible_distribution_major_version == '2')) %}
 User={{ user_name }}
 Group={{ user_name }}
 {% endif %}
-# Start
 ExecStartPre={{yb_home_dir}}/bin/clock-sync.sh
 ExecStart={{yb_home_dir}}/master/bin/yb-master --flagfile {{yb_home_dir}}/master/conf/server.conf
 Restart=always

managed/devops/roles/provision-cluster-server/templates/yb-tserver.service → managed/devops/roles/configure-cluster-server/templates/yb-tserver.service

@@ -11,11 +11,11 @@ PathExists={{yb_home_dir}}/tserver/bin/yb-tserver
 PathExists={{yb_home_dir}}/tserver/conf/server.conf
 
 [Service]
+# Start
 {% if ansible_os_family == 'RedHat' and (ansible_distribution_major_version == '7' or (ansible_distribution == 'Amazon' and ansible_distribution_major_version == '2')) %}
 User={{ user_name }}
 Group={{ user_name }}
 {% endif %}
-# Start
 ExecStartPre={{yb_home_dir}}/bin/clock-sync.sh
 ExecStart={{yb_home_dir}}/tserver/bin/yb-tserver --flagfile {{yb_home_dir}}/tserver/conf/server.conf
 Restart=always

managed/devops/roles/manage_otel_collector/tasks/main.yml

@@ -5,19 +5,13 @@
 
 - name: Install OpenTelemetry collector
   include: install-otel-col.yml
-  become: yes
-  become_method: sudo
   become_user: "{{ user_name }}"
   when: install_otel_collector is defined and install_otel_collector|bool
 
 - name: Create OpenTelemetry collector service
   include: otel-col-service.yml
   when: install_otel_collector is defined and install_otel_collector|bool
-  become: yes
-  become_method: sudo
 
 - name: Control OpenTelemetry collector service
-  include: otel-col-control.yml
-  become: yes
-  become_method: sudo
   become_user: "{{ user_name }}"
+  include: otel-col-control.yml

managed/devops/roles/manage_otel_collector/tasks/otel-col-service.yml

@@ -29,6 +29,7 @@
         state: present
         mode: 0440
         validate: 'visudo -cf %s'
+      when: systemd_dir == "/etc/systemd/system"
 
     # Adding systemd service file for otel-collector
     - name: OpenTelemetry Collector Service | Add otel-collector.service
@@ -51,4 +52,4 @@
       systemd:
         daemon_reload: yes
         scope: user
-      when: systemd_dir != "/etc/systemd/system"
+      when: systemd_dir != "/etc/systemd/system"