1:   2:   3:   4:   5:   6:   7:   8:   9:  10:  11:  12:  13:  14:  15:  16:  17:  18:  19:  20:  21:  22:  23:  24:  25:  26:  27:  28:  29:  30:  31:  32:  33:  34:  35:  36:  37:  38:  39:  40:  41:  42:  43:  44:  45:  46:  47:  48:  49:  50:  51:  52:  53:  54:  55:  56:  57:  58:  59:  60:  61:  62:  63:  64:  65:  66:  67:  68:  69:  70:  71:  72:  73:  74:  75:  76:  77:  78:  79:  80:  81:  82:  83:  84:  85:  86:  87:  88:  89:  90:  91:  92:  93:  94:  95:  96:  97:  98:  99: 100: 101: 102: 103: 104: 105: 106: 107: 108: 109: 110: 111: 112: 113: 114: 115: 116: 117: 118: 119: 120: 121: 122: 123: 124: 125: 126: 127: 128: 129: 130: 131: 132: 133: 134: 135: 136: 137: 138: 139: 140: 141: 142: 143: 144: 145: 146: 147: 148: 149: 150: 151: 152: 153: 154: 155: 156: 157: 158: 159: 160: 161: 162: 163: 164: 165: 166: 167: 168: 169: 170: 171: 172: 173: 174: 175: 176: 177: 178: 179: 180: 181: 182: 183: 184: 185: 186: 187: 188: 189: 190: 191: 192: 193: 194: 195: 196: 197: 198: 199: 200: 201: 202: 203: 204: 205: 206: 207: 208: 209: 210: 211: 212: 213: 214: 215: 216: 217: 218: 219: 220: 221: 222: 223: 224: 225: 226: 227: 228: 229: 230: 231: 232: 233: 234: 235: 236: 237: 238: 239: 240: 241: 242: 243: 244: 245: 246: 247: 248: 249: 250: 251: 252: 253: 254: 255: 256: 257: 258: 259: 260: 261: 262: 263: 264: 265: 266: 267: 268: 269: 270: 271: 272: 273: 274: 275: 276: 277: 278: 279: 280: 281: 282: 283: 284: 285: 286: 287: 288: 289: 290: 291: 292: 293: 294: 295: 296: 297: 298: 299: 300: 301: 302: 303: 304: 305: 306: 307: 308: 309: 310: 311: 312: 313: 314: 315: 316: 317: 318: 319: 320: 321: 322: 323: 324: 325: 326: 327: 328: 329: 330: 331: 
<?php
    declare(strict_types=1);
    /**
     *  +------------------------------------------------------------+
     *  | apnscp                                                     |
     *  +------------------------------------------------------------+
     *  | Copyright (c) Apis Networks                                |
     *  +------------------------------------------------------------+
     *  | Licensed under Artistic License 2.0                        |
     *  +------------------------------------------------------------+
     *  | Author: Matt Saladna (msaladna@apisnetworks.com)           |
     *  +------------------------------------------------------------+
     */

    use Daphnie\Collector;
    use Opcenter\System\Cgroup;

    /**
     *  Control group interfacing
     *
     * @package core
     */
    class Cgroup_Module extends Module_Skeleton implements \Opcenter\Contracts\Hookable
    {
        const CGROUP_LOCATION = Cgroup::CGROUP_HOME;
        const DEPENDENCY_MAP = [
            'siteinfo'
        ];
        const DEFAULT_MEMORY = 512;
        const DEFAULT_CPU = 10240;
        /** in MB */
        const MAX_PROCS = 25;

        const METRIC_ATTR_CPU_USAGE = [
            'c-cpuacct-usage',
            'c-cpuacct-system',
            'c-cpuacct-user'
        ];

        protected $exportedFunctions = [
            '*' => PRIVILEGE_SITE | PRIVILEGE_USER | PRIVILEGE_ADMIN
        ];

        /**
         * Get controller usage
         *
         * @param string $controller
         * @return array|bool
         */
        public function get_usage(string $controller)
        {
            if (!IS_CLI) {
                return $this->query('cgroup_get_usage', $controller);
            }
            if (!in_array($controller, $this->get_controllers(), true)) {
                return error("unknown controller `%s'", $controller);
            }

            return $this->{'_get_' . $controller . '_usage'}();
        }

        /**
         * Get cgroup controllers
         *
         * @return string[]
         */
        public function get_controllers(): array
        {
            return CGROUP_CONTROLLERS;
        }

        /**
         * Get cgroup name
         *
         * @return string|null
         */
        public function get_cgroup(): ?string
        {
            if ($this->permission_level & (PRIVILEGE_SITE | PRIVILEGE_USER)) {
                return $this->site;
            }

            return null;
        }

        /**
         * Get configured limits
         *
         * @return array
         */
        public function get_limits(): array
        {
            $limits = $this->getServiceValue('cgroup');
            if (!$limits['enabled']) {
                return [];
            }

            return array_except($limits, ['version', 'enabled']);
        }

        /**
         * cgroups enabled for site
         *
         * @return bool
         */
        public function enabled(): bool
        {
            return (bool)$this->getServiceValue('cgroup', 'enabled');
        }

        public function _verify_conf(\Opcenter\Service\ConfigurationContext $ctx): bool
        {
            return true;
        }

        public function _create()
        { }

        public function _delete()
        { }

        public function _edit()
        { }

        public function _create_user(string $user)
        {
            return true;
        }

        public function _delete_user(string $user)
        {
            return true;
        }

        public function _edit_user(string $userold, string $usernew, array $oldpwd)
        {
            return true;
        }

        public function _housekeeping()
        {
            if (!($test = $this->get_controllers()[0] ?? null)) {
                return;
            }
            if (!\Opcenter\Filesystem\Mount::mounted(FILESYSTEM_SHARED . "/cgroup/${test}") && !Cgroup::mountAll()) {
                return false;
            }
            $webuser = $this->web_get_sys_user();
            foreach (\Opcenter\Account\Enumerate::sites() as $site) {
                if (!Auth::get_admin_from_site_id((int)substr($site, 4))) {
                    continue;
                }
                $group = new \Opcenter\System\Cgroup\Group(
                    $site,
                    [
                        'task' => [
                            'uid' => $webuser,
                            'gid' => \Auth::get_group_from_site($site)
                        ]
                    ]
                );
                $ctx = $afi = null;
                foreach (Cgroup::getControllers() as $c) {
                    $controller = \Opcenter\System\Cgroup\Controller::make($group, $c, []);
                    if (Cgroup::exists($controller, $group)) {
                        continue;
                    }
                    if (null === $ctx) {
                        $ctx = \Auth::context(null, $site);
                        $afi = \apnscpFunctionInterceptor::factory($ctx);
                    }
                    $controller->import($afi, $ctx);
                    $controller->create();
                    $group->add($controller);
                }
                Cgroup::create($group);
            }

            return true;
        }

        /**
         * Get controller memory usage
         *
         * @return array
         */
        private function _get_memory_usage(): array
        {
            $stats['limit'] = self::DEFAULT_MEMORY;
            $stats = Cgroup::memory_usage($this->site);
            $sysMemory = \Opcenter\System\Memory::stats();
            $maxMemory = $sysMemory['memtotal'] * 1024;
            if ($this->permission_level & PRIVILEGE_ADMIN || $stats['limit'] === null) {
                $stats['limit'] = $maxMemory;
                $stats['free'] = $sysMemory['memavailable']*1024;
            } else {
                $stats['limit'] = min($stats['limit'], $maxMemory);
                $stats['free'] = $stats['limit'] - $stats['used'];
            }

            return $stats;
        }

        /**
         * Populate cgroup defaults on controller error
         *
         * @param array $usage
         * @param array $defaults
         * @return array
         */
        private function _fillUsage(array $usage, array $defaults): array
        {
            foreach ($defaults as $k => $v) {
                if (!isset($usage[$k])) {
                    $usage[$k] = $v;
                }
            }

            return $usage;
        }

        private function _get_cpuacct_usage(): array
        {
            return [];
        }

        private function _get_pids_usage(): array
        {
            // @todo replace CPU maxproc with pids subsystem
            $maxprocs = self::MAX_PROCS;
            if ($this->permission_level & PRIVILEGE_ADMIN) {
                $maxprocs = 999;
            }

            return $this->_fillUsage(
                Cgroup::pid_usage($this->site),
                [
                    'max' => $this->getServiceValue('cgroup', 'proclimit', $maxprocs)
                ]
            );
        }

        private function _get_cpu_usage(): array
        {
            $maxcpu = self::DEFAULT_CPU;
            $maxprocs = self::MAX_PROCS;
            if ($this->permission_level & PRIVILEGE_ADMIN) {
                $maxcpu = NPROC * 86400;
                $maxprocs = 999;
            }

            $usage = Cgroup::cpu_usage($this->site);
            if (($this->permission_level & PRIVILEGE_SITE) && TELEMETRY_ENABLED) {
                $sum = $this->telemetry_range(self::METRIC_ATTR_CPU_USAGE, time()-86400, null, $this->site_id, true);
                /**
                 * > .usage is measuring the wall clock nanoseconds whereas .stat is measuring the cpu cycles consumed.
                 * http://mail-archives.apache.org/mod_mbox/mesos-dev/201302.mbox/%3C20130214015558.21380.50889@reviews.apache.org%3E
                 */
                // convert centiseconds to seconds
                $cumusage = ($sum['c-cpuacct-usage'] ?? 0)/100;
                $usage['cumusage'] = $usage['used'];
                $usage['used'] = $cumusage ?: $usage['used'];
                $usage['cumuser'] = $usage['user'];
                $usage['cumsystem'] = $usage['system'];
                $usage['system'] = ($sum['c-cpuacct-system'] ?? 0)/100 ;
                $usage['user'] = ($sum['c-cpuacct-user'] ?? 0)/ 100;
            }
            $cpuLimit = $this->getServiceValue('cgroup', 'cpu', $maxcpu);
            return $this->_fillUsage(
                $usage,
                [
                    'limit'    => $cpuLimit,
                    'maxprocs' => $this->getServiceValue('cgroup', 'proclimit', $maxprocs),
                    'cumusage' => $usage['used'],
                    'free'     => $cpuLimit - $usage['used']
                ]
            );
        }

        private function _get_blkio_usage(): array
        {
            return $this->_fillUsage(
                Cgroup::io_usage($this->site),
                [
                    'iops-read'  => $this->getServiceValue('cgroup', 'readiops', 100),
                    'iops-write' => $this->getServiceValue('cgroup', 'writeiops', 100),
                    'bw-read'    => $this->getServiceValue('cgroup', 'readbw', 100),
                    'bw-write'   => $this->getServiceValue('cgroup', 'writebw', 100)
                ]
            );
        }

        public function _cron(Cronus $cron) {
            if (!TELEMETRY_ENABLED) {
                return;
            }
            $db = PostgreSQL::pdo();
            $collector = new Collector($db);
            // read from siteinfo table to guard protect against failed foreign key checks
            $sites = (new \Opcenter\Database\PostgreSQL\Opcenter($db))->readSitesFromSiteinfo();
            $sites[] = null; // system controller
            $controllers = $this->get_controllers();
            foreach (array_keys($sites) as $s) {
                $s = "site${s}";
                $siteId = $s === null ? null : (int)substr($s, 4);
                $ts = time();
                /**
                 * Approx 32k controllers/sec on testing VM (~5500 backend req/sec)
                 * This method should be fine with minimal performance degradation,
                 * may wish to switch to less OO approach in the future if bottlenecks appear
                 *
                 * Takes ~5ms to log all metrics for a site
                 */
                $group = new Cgroup\Group($s);
                $counters = [];
                foreach ($controllers as $c) {
                    $controller = Cgroup\Controller::make($group, $c);
                    $logger = (new Cgroup\MetricsLogging($controller));
                    $attrs = $logger->getLoggableAttributes();
                    $counters[$c] = $controller->readCounters(array_keys($attrs));
                    foreach ($counters[$c] as $k => $v) {
                        $collector->add($attrs[$k], $siteId, (int)$v, $ts);
                    }
                }
            }
            $collector = null;

            return true;
        }
    }