Devil Killer Is Here MiNi Shell
MiNi SheLL

Current Path : /home/vmanager/www/console/controllers/
Linux 9dbcd5f6333d 5.15.0-102-generic #112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024 x86_64
Current File : /home/vmanager/www/console/controllers/TendersController.php
<?php

namespace console\controllers;

use \Yii;
use \yii\console\Controller;
use \yii\helpers\Console;
use yii\console\ExitCode;

class TendersController extends Controller
{
    const LIMIT_FOR_PROCESSING = 200;

    private $errorMessage;

    public function actionDownload()
    {        
        $lastDate = $this->getLastFileDate();
//        var_dump($lastDate->format('Y-m-d')); exit;
        //proteza
        $lastDate = \DateTime::createFromFormat('Y-m-d', '2019-01-01');
        
        $nextDate = clone $lastDate;
//        $todayDate = new \DateTime();

        $dirHTML = Yii::getAlias('@runtime/temp/tenders/html/');
        if(!file_exists($dirHTML)) {
            mkdir($dirHTML, 0777, true);
        }

        $nextDate->modify('+1 day');
        $filename = $nextDate->format('Ymd').'.exe';

//        if($nextDate < $todayDate) {
            $fileHTMLUrl = 'ftp://ftp.uzp.gov.pl/bzp2/html/'.$nextDate->format('Y').'/'.$filename;
            $fileDate = $nextDate->format('Y-m-d');
//        } 
//        else {
//            $fileHTMLUrl = 'ftp://ftp.uzp.gov.pl/bzp2/dzis/html/dzishtml.exe';
//            $fileDate = $todayDate->format('Y-m-d');
//        }

        try {
            $fileHTMLContent = file_get_contents($fileHTMLUrl);
        }
        catch (\Exception $e) {
//            $this->stderr('No file: '.$fileHTMLUrl);
            return ExitCode::NOINPUT;
        }
        file_put_contents($dirHTML.$filename, $fileHTMLContent);

        $this->saveActualFileDate($fileDate);
        
        //wypakowywanie
        $archive = \RarArchive::open($dirHTML.$filename);
        $entries = $archive->getEntries();
        if($entries === FALSE) {
//            $this->stderr('No files in archive: '.$dirHTML.$filename);
            return ExitCode::DATAERR;
        }
        foreach($entries as $entry) {
            $entry->extract($dirHTML.$nextDate->format('Ymd').'/');
        }
	$archive->close();

        return ExitCode::OK;
    }
    
    public function actionImport()
    {        
        $actualDatetime = date('Y-m-d H:i:s');

        $dirName = $this->getSourceDirName();
        if($dirName === false) {
//            $this->stderr(!empty($this->errorMessage) ? $this->errorMessage : 'No files for processing. Possible error occured while downloading archive with sources.');
            return ExitCode::NOINPUT;
        }
        
        $processingDate = substr_replace($dirName, '-', 4, 0);
        $processingDate = substr_replace($processingDate, '-', 7, 0);
        
//        $existsTanders = $this->getExistsTendersForDate($processingDate); //@todo UWAGA !!! Wydaje mi się, że sprawdzanie wyłącznie w obrębie danej daty doprowadzi 
        //do duplikatu klucza głównego w momencie, gdy dany przetarg wystąpi dwukrotnie (np. będzie siedział w plikach dla dwóch następujących po sobie dni)    
        
        $dirHTML = Yii::getAlias('@runtime/temp/tenders/html/').$dirName;
        $directoryIterator = new \DirectoryIterator($dirHTML);
        $i = 0;
        foreach ($directoryIterator as $item) {
            if ($item->isFile() && $item->getExtension() === 'html') {
                $data = $this->parseTender($item->getPathname());
                if($data === false) {
                    @unlink($item->getPathname());
                    continue;
                }
                
                $i++;
                if($i > self::LIMIT_FOR_PROCESSING) {
//                    $this->stdout('Exceeded the limit of elements processed in one call.');
                    return ExitCode::OK;
                }

                try {
//                    if(!isset($existsTanders[$data['_id']])) {
                    $orExist = (new \yii\mongodb\Query())->from('tenders')->where(['_id' => $data['_id']])->exists();
                    if(!$orExist) {
                        $data['or_hidden_by_inquiry'] = 0;
                        $data['created_at'] = $data['updated_at'] = new \MongoDB\BSON\UTCDateTime(strtotime($actualDatetime)*1000);
                        
                        Yii::$app->mongodb->getCollection('tenders')->insert($data); 
                        $this->saveHTMLFile($dirName, $item->getBasename());
                    }
                }
                catch (\yii\mongodb\Exception $exception) {
//                    var_dump($exception->getMessage());
//                    $this->stderr('Error writing to the database.');
                    return ExitCode::UNSPECIFIED_ERROR;
                }

                //dopisać wstawiony dokument do listy obecnych przetargów dla wybranej daty
//                $existsTanders[$data['_id']] = true;

                @unlink($item->getPathname());
            }
        }
        
        \yii\caching\TagDependency::invalidate(Yii::$app->frontendCache, ['mainpage_box_tenders']);
    }
    
    private function prepareText($text) 
    {
	$text = str_replace('&quot;','"',$text);
	$text = str_replace("'",'&rsquo;',$text);
	$text = str_replace("",'&rsquo;',$text);
        
	return $text;
    }
    
    protected function parseTender($path)
    {
        $fileBaseName = basename($path);
        if (strpos($fileBaseName, 'N') > 0) {
            $fileContent = file_get_contents($path);
            $matches = array();
            preg_match('#<span id="ctl00_ContentPlaceHolder1_lblNumerOgloszenia">.*([0-9]{4}\-[0-9]{2}\-[0-9]{2}) r. </span>#i', $fileContent, $matches);
            if(empty($matches[1])) {
                unset($fileContent);
                return false;
            } else {
                $data = [];
                $data['data_publikacji'] = trim($matches[1]);
                $data['biuletyn'] = date('Y', strtotime($data['data_publikacji']));
            }
        } else {
            return false;
        }   
        
        $data['file_html'] = $fileBaseName;
        
        $pathinfo = pathinfo($path, PATHINFO_DIRNAME);
        $pathinfo = array_filter(explode('/', $pathinfo));
        $data['dir_html'] = end($pathinfo); 
        
        $data['pozycja'] = substr($data['file_html'], 0, -5);
//        $data['rodzaj_zamowienia'] = 0; //nie wiem dlaczego, tak jest w bazie...


//        $matches = array();
//        $data['data_publikacji = '1970-01-01';
//        preg_match('#<span id="ctl00_ContentPlaceHolder1_lblNumerOgloszenia">.*([0-9]{4}\-[0-9]{2}\-[0-9]{2}) r. </span>#i', $fileContent, $matches);
//        if(!empty($matches[1])) {
//            $data['data_publikacji = trim($matches[1]);
//        }

        $matches = array();
        $data['nazwa_zamowienia'] = '';
        preg_match('#<span id="ctl00_ContentPlaceHolder1_nazwa_nadana_zamowieniu_h">(.*)</span>#i', $fileContent, $matches);
        if(!empty($matches[1])) {
            $data['nazwa_zamowienia'] = trim($this->prepareText($matches[1]));
        }

        $matches = array();
        $data['nazwa'] = '';
        preg_match('#<span id="ctl00_ContentPlaceHolder1_zamawiajacy_nazwa">(.*)</span>#i', $fileContent, $matches);
        if(!empty($matches[1])) {
            $data['nazwa'] = trim($this->prepareText($matches[1]));
        }

        $matches = array();
        $data['ulica'] = '';
        preg_match('#<span id="ctl00_ContentPlaceHolder1_zamawiajacy_adres_ulica">(.*)</span>#i', $fileContent, $matches);
        if(!empty($matches[1])) {
            $data['ulica'] = trim($this->prepareText($matches[1]));
        }

        $matches = array();
        $data['nr_domu'] = '';
        preg_match('#<span id="ctl00_ContentPlaceHolder1_zamawiajacy_adres_numer_domu">(.*)</span>#i', $fileContent, $matches);
        if(!empty($matches[1])) {
            $data['nr_domu'] = trim($this->prepareText($matches[1]));
        }

        $matches = array();
        $data['nr_miesz'] = '';
        preg_match('#<span id="ctl00_ContentPlaceHolder1_zamawiajacy_adres_numer_mieszkania">(.*)</span>#i', $fileContent, $matches);
        if(!empty($matches[1])) {
            $data['nr_miesz'] = trim($this->prepareText($matches[1]));
        }

        $matches = array();
        $data['miejscowosc'] = '';
        preg_match('#<span id="ctl00_ContentPlaceHolder1_zamawiajacy_miejscowosc">(.*)</span>#i', $fileContent, $matches);
        if(!empty($matches[1])) {
            $data['miejscowosc'] = trim($this->prepareText($matches[1]));
        }

        $matches = array();
        $data['kod_poczt'] = '';
        preg_match('#<span id="ctl00_ContentPlaceHolder1_zamawiajacy_kod_pocztowy">(.*)</span>#i', $fileContent, $matches);
        if(!empty($matches[1])) {
            $data['kod_poczt'] = trim($matches[1]);
            if(strpos($data['kod_poczt'], "-") === false) {
                $data['kod_poczt'] = substr($data['kod_poczt'], 0, 2).'-'.substr($data['kod_poczt'], 2);
            }
        }

        $matches = array();
        $data['wojewodztwo'] = '';
//        $data['woj'] = 0;
        preg_match('#<span id="ctl00_ContentPlaceHolder1_zamawiajacy_wojewodztwo">(.*)</span>#i', $fileContent, $matches);
        if(!empty($matches[1])) {
            $data['wojewodztwo'] = trim($this->prepareText($matches[1]));
//            $data['woj'] = $idWoj[$data['wojewodztwo']];
        }

        $matches = array();
        $data['tel'] = '';
        preg_match('#<span id="ctl00_ContentPlaceHolder1_zamawiajacy_telefon">(.*)</span>#i', $fileContent, $matches);
        if(!empty($matches[1])) {
            $data['tel'] = trim($this->prepareText($matches[1]));
        }

        $matches = array();
        $data['fax'] = '';
        preg_match('#<span id="ctl00_ContentPlaceHolder1_zamawiajacy_fax">(.*)</span>#i', $fileContent, $matches);
        if(!empty($matches[1])) {
            $data['fax'] = trim($this->prepareText($matches[1]));
        }

        $matches = array();
        $data['email'] = '';
        preg_match('#<span id="ctl00_ContentPlaceHolder1_zamawiajacy_email">(.*)</span>#i', $fileContent, $matches);
        if(!empty($matches[1])) {
            $data['email'] = trim($matches[1]);
        }

        $matches = array();
        $data['www'] = '';
        preg_match('#<span id="ctl00_ContentPlaceHolder1_adres_strony_url">(.*)</span>#i', $fileContent, $matches);
        if(!empty($matches[1])) {
            $data['www'] = trim($matches[1]);
        }

        $matches = array();
        $data['numercpv'] = '';
        preg_match('#<span id="ctl00_ContentPlaceHolder1_cpv_glowny_przedmiot">(.*)</span>#i', $fileContent, $matches);
        if(!empty($matches[1])) {
//            $data['numercpv'] = substr(trim($matches[1]), 0, 8);
            $data['numercpv'] = substr(trim($matches[1]), 0, 8);
        }

        $matches = array();
        $data['numercpv2'] = '';
        preg_match('#<TR><TD style=\'border: 1px solid black;border-collapse: collapse;\'>Kod CPV</TD></TR><TR><TD style=\'border: 1px solid black;border-collapse: collapse;\'>([0-9]{8}-[0-9]{1})</TD></TR>#i', $fileContent, $matches);
        if(!empty($matches[1])) {
//            $data['numercpv2'] = substr(trim($matches[1]), 0, 8);
            $data['numercpv2'] = substr(trim($matches[1]), 0, 8);
        }    

        $data['_id'] = substr($data['data_publikacji'], 2, 2).substr($data['pozycja'], 0, strpos($data['pozycja'], '-'));
        $data['data_publikacji'] = new \MongoDB\BSON\UTCDateTime(strtotime($data['data_publikacji'])*1000);
//        $data['updated_at'] = new \MongoDB\BSON\UTCDateTime();
        
        return $data;
    }
    
    protected function getSourceDirName()
    {
        $directoryIterator = new \DirectoryIterator(Yii::getAlias('@runtime/temp/tenders/html/'));
        $dirName = null;
        foreach ($directoryIterator as $item) {
            if (!$item->isDot() && $item->isDir()) {
                $dirName = $item->getFilename();
                $directoryIterator2 = new \DirectoryIterator(Yii::getAlias('@runtime/temp/tenders/html/'.$dirName));
                $dirOk = false;
                foreach ($directoryIterator2 as $item2) {
                    if ($item2->isFile() && $item2->getExtension() === 'html') {
                        $dirOk = true;
                        break;
                    }
                }
                if($dirOk) {
                    break;
                } else {
                    $this->removeDir(Yii::getAlias('@runtime/temp/tenders/html/').$dirName);
                    $dirName = null;
                    continue;
                }
            }
        }

        if($dirName === null) {
            $dirName = $this->downloadNextArchive();            
        }        

        return (!empty($dirName)) ? $dirName : false;
    }
    
//    protected function getExistsTendersForDate($processingDate)
//    {
//        $query = new \yii\mongodb\Query();
//        $rows = $query->from('tenders')->select(['_id'])->where(['data_publikacji' => new \MongoDB\BSON\UTCDateTime(strtotime($processingDate)*1000)])->all();
//        $existsTanders = [];
//        if(!empty($rows)) {
//            foreach($rows as $row) {
//                $existsTanders[$row['_id']] = true;
//            }
//        } 
//        
//        return $existsTanders;
//    }
    
    protected function parseTender2($xml)
    {
        $actualDatetime = date('Y-m-d H:i:s');
        
        $xmlData = json_decode(json_encode($xml), true);
        $xmlData['_id'] = (string)substr($xmlData['data_publikacji'], 2, 2).substr($xmlData['pozycja'], 0, strpos($xmlData['pozycja'], '-'));
        $xmlData['data_publikacji'] = new \MongoDB\BSON\UTCDateTime(strtotime($xmlData['data_publikacji'])*1000);
        $xmlData['updated_at'] = new \MongoDB\BSON\UTCDateTime(strtotime($actualDatetime)*1000);
        
        return $xmlData;
    }
    
    protected function saveHTMLFile($dirName, $filename)
    {
        $source = Yii::getAlias('@runtime/temp/tenders/html/').$dirName.'/'.$filename;
        $target = Yii::getAlias('@tendersRealPath').'/'.$dirName.'/'.$filename;
        
        if(!file_exists($source)) {
            return false;
        }
        if(!file_exists(Yii::getAlias('@tendersRealPath').'/'.$dirName.'/')) {
            mkdir(Yii::getAlias('@tendersRealPath').'/'.$dirName.'/', 0777, true);
        }
        
        $fileContent = file_get_contents($source);
        $matches = [];
        preg_match_all('~<body[^>]*>(.*?)<\/body>~is', $fileContent, $matches);
        if(!empty($matches[1])) {
            $fileContent = $matches[1];
        }        
        $fileContent = preg_replace('|<script.*</script>|i', '', $fileContent);
        //$fileContent = preg_replace('|<style type="text/css">.*</style>|is', '', $fileContent);
        $fileContent = preg_replace('|<input.*>|i', '', $fileContent);
        $fileContent = preg_replace('|<form .*>|i', '', $fileContent);
        $fileContent = preg_replace('|</form>|i', '', $fileContent);
        $fileContent = str_replace('<link href="../xslt.css" rel="stylesheet" type="text/css" />', '', $fileContent);
        $fileContent = preg_replace("|if \(typeof\(Sys\) === 'undefined'\) throw new Error\(.*\);|i", '', $fileContent);
        $fileContent = str_replace('(function() {var fn = function() {$get("ctl00_ToolkitScriptManager1_HiddenField").value = \'\';Sys.Application.remove_init(fn);};Sys.Application.add_init(fn);})();', '', $fileContent);
        //odkomentować, jeśli ma wywalić również style inline
        //$fileContent = preg_replace('|style=".*"|i', '', $fileContent);
        $fileContent = preg_replace('/[\n\r]+/', '', $fileContent);
        $fileContent = preg_replace('/[\s]{2,}/', ' ', $fileContent);
        $flag = file_put_contents($target, $fileContent);
        
        return $flag !== false ? true : false;        
    }
    
    protected function downloadNextArchive()
    {
        $dirHTML = Yii::getAlias('@runtime/temp/tenders/html/');
        if(!file_exists($dirHTML)) {
            mkdir($dirHTML, 0777, true);
        }
        
        $lastDate = $this->getLastFileDate();
        
        $nextDate = clone $lastDate;
        $nextDate->modify('+1 day');
        $todayDate = new \DateTime();     
        $dateDiff = $nextDate->diff($todayDate);   

        //gdy szukana data jest większa od aktualnej lub mniejsza, ale o mniej niż jeden dzień, to od razu zwracamy false,
        //ponieważ pliki pojawiają się na serwerze z jednodniowym opóźnieniem
        if($dateDiff->invert === 1 || ($dateDiff->invert === 0 && $dateDiff->days < 1)) { 
            return false;
        }
        
        $filename = $nextDate->format('Ymd').'.exe';
        $fileHTMLUrl = 'ftp://ftp.uzp.gov.pl/bzp2/html/'.$nextDate->format('Y').'/'.$filename;
        $fileDate = $nextDate->format('Y-m-d');

        try {
            $fileHTMLContent = file_get_contents($fileHTMLUrl);
        }
        catch (\Exception $e) {
            //jeśli nie ma na serwerze pliku, a szukana data jest starsza o więcej niż dwa dni od aktualnej,
            //to zakładamy, że owy plik już się na serwerze nie pojawi, więc przestawiamy datę synchronizacji
            //właśnie na tą szukaną
            if($dateDiff->invert === 0 && $dateDiff->days > 2) {
                $this->saveActualFileDate($nextDate->format('Y-m-d'));
            }
            
            $this->errorMessage = 'No file: '.$fileHTMLUrl;
            return false;
        }
        
        file_put_contents($dirHTML.$filename, $fileHTMLContent);

        $this->saveActualFileDate($fileDate);
        
        //wypakowywanie        
        $archive = \RarArchive::open($dirHTML.$filename);
        $entries = $archive->getEntries();
        if($entries === FALSE) {
            $this->errorMessage = 'No files in archive: '.$dirHTML.$filename;
            return false;
        }
        $extractDirName = $nextDate->format('Ymd');
        foreach($entries as $entry) {
            $entry->extract($dirHTML.$extractDirName.'/');
        }
	$archive->close();
        @unlink($dirHTML.$filename);

        return $extractDirName;
    }

    protected function removeDir($path)
    {
        $dir = new \DirectoryIterator($path);
        foreach($dir as $fileinfo) {
            if($fileinfo->isFile() || $fileinfo->isLink()) {
                @unlink($fileinfo->getPathName());
            }
            elseif(!$fileinfo->isDot() && $fileinfo->isDir()) {
                $this->removeDir($fileinfo->getPathName());
            }
        }
        unset($dir, $fileinfo);
        rmdir($path);
    }
    
    protected function getLastFileDate()
    {
        $query = new \yii\mongodb\Query();
        $row = $query->from('tenders_import_status')->select(['file_date'])->orderBy('file_date DESC')->one();
        if(!empty($row)) {
            return $row['file_date']->toDateTime()->setTimezone(new \DateTimeZone(date_default_timezone_get()));
//            return \DateTime::createFromFormat('Y-m-d', date('Y-m-d', $row['file_date']->sec));
        } else {
            //@todo tutaj trzeba będzie ustawić datę, od której zaczynamy import            
            return new \DateTime('2019-10-25');
        }
    }
    
    protected function saveActualFileDate($fileDate)
    {
        try {
            $fileDate = new \MongoDB\BSON\UTCDateTime(strtotime($fileDate)*1000); 
            $actualDatetime = new \MongoDB\BSON\UTCDateTime();
            Yii::$app->mongodb->getCollection('tenders_import_status')->save(['file_date' => $fileDate, 'created_at' => $actualDatetime]);

        }
        catch (\yii\mongodb\Exception $exception) {
//            $this->stderr('Error writing to the database.');
            return Controller::EXIT_CODE_ERROR;
        }
        
    }
}
Creat By MiNi SheLL
Email: jattceo@gmail.com