Automation init...3

This commit is contained in:
최준흠 2024-09-16 19:44:45 +09:00
parent 6ad2ecedba
commit 6305932285
6 changed files with 134 additions and 138 deletions

View File

@ -15,7 +15,7 @@ abstract class MyCrawler extends CommonLibrary
parent::__construct(); parent::__construct();
$this->_mySocket = $mySocket; $this->_mySocket = $mySocket;
} }
abstract protected function getMyStorage(); abstract protected function createMyStorage();
abstract protected function detail_page(int $cnt, array $listInfo): void; abstract protected function detail_page(int $cnt, array $listInfo): void;
final protected function getMySocket() final protected function getMySocket()
{ {
@ -61,7 +61,7 @@ abstract class MyCrawler extends CommonLibrary
log_message("debug", "\n-----------detailPage Tag: {$tag}---------------\n{$selector->html()}\n---------------------------\n"); log_message("debug", "\n-----------detailPage Tag: {$tag}---------------\n{$selector->html()}\n---------------------------\n");
$urls = $this->getMediaUrlsByMediaType("image", $selector, ["tag" => "img", "attr" => "src"]); $urls = $this->getMediaUrlsByMediaType("image", $selector, ["tag" => "img", "attr" => "src"]);
$urls = $this->getMediaUrlsByMediaType("video", $selector, ["tag" => "video", "attr" => "src"], $urls); $urls = $this->getMediaUrlsByMediaType("video", $selector, ["tag" => "video", "attr" => "src"], $urls);
log_message("debug", "\n-------------------------\n" . var_export($urls, true) . "\n-----------------------\n"); // log_message("debug", "\n-------------------------\n" . var_export($urls, true) . "\n-----------------------\n");
log_message("notice", "-----------" . __FUNCTION__ . " 작업완료--------"); log_message("notice", "-----------" . __FUNCTION__ . " 작업완료--------");
return array($listInfo, $urls); return array($listInfo, $urls);
} }
@ -70,11 +70,12 @@ abstract class MyCrawler extends CommonLibrary
private function media_save(int $file_sequence, string $media_type, string $file_name, string $content): mixed private function media_save(int $file_sequence, string $media_type, string $file_name, string $content): mixed
{ {
log_message("debug", __FUNCTION__ . " 원본파일 {$file_name} 작업 시작"); log_message("debug", __FUNCTION__ . " 원본파일 {$file_name} 작업 시작");
$this->getMyStorage()->setOriginName($file_name); $storage = $this->createMyStorage();
$this->getMyStorage()->setOriginContent($content); $storage->setOriginName($file_name);
$this->getMyStorage()->setOriginType($media_type); $storage->setOriginContent($content);
$this->getMyStorage()->setOriginSequence($file_sequence); $storage->setOriginType($media_type);
return $this->getMyStorage()->save(); $storage->setOriginSequence($file_sequence);
return $storage->save();
} }
//Yamap ViewPage의 이미지나영상데이터가 있으면 Dodownload 한다. //Yamap ViewPage의 이미지나영상데이터가 있으면 Dodownload 한다.
private function media_download(string $media_type, string $url): array private function media_download(string $media_type, string $url): array
@ -95,7 +96,7 @@ abstract class MyCrawler extends CommonLibrary
} }
final protected function media_process(array $media_urls): array final protected function media_process(array $media_urls): array
{ {
log_message("debug", var_export($media_urls, true)); // log_message("debug", var_export($media_urls, true));
$file_sequence = 1; $file_sequence = 1;
$storages = []; //CreateBoard에서 사용을 위해 DetailPage마다 초기화 $storages = []; //CreateBoard에서 사용을 위해 DetailPage마다 초기화
// log_message("debug", var_export($urls, true)); // log_message("debug", var_export($urls, true));
@ -108,7 +109,7 @@ abstract class MyCrawler extends CommonLibrary
log_message("notice", __FUNCTION__ . " MediaType->{$media_type} 작업 완료"); log_message("notice", __FUNCTION__ . " MediaType->{$media_type} 작업 완료");
} catch (\Exception $e) { } catch (\Exception $e) {
log_message("warning", sprintf( log_message("warning", sprintf(
"\n---%s mediaType->%s 오류---\n%s\n-----------------------------------------\n", "\n---%s MediaType->%s 오류---\n%s\n-----------------------------------------\n",
__FUNCTION__, __FUNCTION__,
$media_type, $media_type,
$e->getMessage() $e->getMessage()
@ -116,6 +117,11 @@ abstract class MyCrawler extends CommonLibrary
} }
} }
} }
$i = 1;
foreach ($storages as $storage) {
log_message("debug", __FUNCTION__ . " 결과{$i}:" . $storage->getOriginName());
$i++;
}
return $storages; return $storages;
} }
protected function main_process(int $max_limit, array $listInfos): void protected function main_process(int $max_limit, array $listInfos): void

View File

@ -7,28 +7,21 @@ use App\Libraries\MySocket\WebSocket;
use App\Libraries\MyStorage\MangboardStorage; use App\Libraries\MyStorage\MangboardStorage;
use App\Models\Mangboard\BoardModel; use App\Models\Mangboard\BoardModel;
use App\Models\Mangboard\BoardsModel; use App\Models\Mangboard\BoardsModel;
use App\Models\Mangboard\FileModel;
use App\Traits\ImageTrait;
use Symfony\Component\DomCrawler\Crawler; use Symfony\Component\DomCrawler\Crawler;
class YamapCrawler extends MyCrawler class YamapCrawler extends MyCrawler
{ {
use ImageTrait;
private $_category = ""; private $_category = "";
private $_user_entity = null; private $_user_entity = null;
private $_myStorage = null;
public function __construct(string $host, string $category, UserEntity $user_entity) public function __construct(string $host, string $category, UserEntity $user_entity)
{ {
parent::__construct(new WebSocket($host)); parent::__construct(new WebSocket($host));
$this->_category = $category; $this->_category = $category;
$this->_user_entity = $user_entity; $this->_user_entity = $user_entity;
} }
final protected function getMyStorage() final protected function createMyStorage()
{ {
if ($this->_myStorage === null) { return new MangboardStorage($this->_category, $this->_user_entity);
$this->_myStorage = new MangboardStorage($this->_category);
}
return $this->_myStorage;
} }
//작성내용 //작성내용
// <div class="panel panel-default"> // <div class="panel panel-default">
@ -74,9 +67,6 @@ class YamapCrawler extends MyCrawler
$baord_name = "board_" . $this->_category; $baord_name = "board_" . $this->_category;
$boardsModel = new BoardsModel(); $boardsModel = new BoardsModel();
$boards_entity = $boardsModel->getEntityByID("board_" . $this->_category); $boards_entity = $boardsModel->getEntityByID("board_" . $this->_category);
if ($boards_entity === null) {
throw new \Exception("boards에서 {$baord_name} 해당정보를 찾을수 없습니다.");
}
$boardModel = new BoardModel("mb_" . $baord_name); $boardModel = new BoardModel("mb_" . $baord_name);
$board_entity = $boardModel->createByCrawler( $board_entity = $boardModel->createByCrawler(
$boards_entity, $boards_entity,
@ -85,18 +75,20 @@ class YamapCrawler extends MyCrawler
$listInfo, $listInfo,
$storages $storages
); );
if ($board_entity === null) { foreach ($storages as $storage) {
throw new \Exception("{$baord_name} 생성에 오류가 발생했습니다."); try {
$storage->backend($boards_entity, $board_entity, $boardModel->getTable());
} catch (\Exception $e) {
log_message("notice", sprintf(
"\n---%s -> %s 게시물의 %s번째:%s 파일 등록 오류---\n%s\n--------------------------------\n",
__FUNCTION__,
$board_entity->getTitle(),
$storage->getOriginSequence(),
$storage->getOriginName(),
$e->getMessage()
));
}
} }
$fileModel = new FileModel();
$fileModel->createByCrawler(
$boards_entity,
$this->_user_entity,
$board_entity,
$boardModel->getTable(),
$storages
);
$this->create_small_ImageTrait($board_entity, $storages);
} }
public function execute(int $max_limit): void public function execute(int $max_limit): void
{ {

View File

@ -7,28 +7,21 @@ use App\Libraries\MySocket\WebSocket;
use App\Libraries\MyStorage\MangboardStorage; use App\Libraries\MyStorage\MangboardStorage;
use App\Models\Mangboard\BoardModel; use App\Models\Mangboard\BoardModel;
use App\Models\Mangboard\BoardsModel; use App\Models\Mangboard\BoardsModel;
use App\Models\Mangboard\FileModel;
use App\Traits\ImageTrait;
use Symfony\Component\DomCrawler\Crawler; use Symfony\Component\DomCrawler\Crawler;
class YamoonCrawler extends MyCrawler class YamoonCrawler extends MyCrawler
{ {
use ImageTrait;
private $_category = ""; private $_category = "";
private $_user_entity = null; private $_user_entity = null;
private $_myStorage = null;
public function __construct(string $host, string $category, UserEntity $user_entity) public function __construct(string $host, string $category, UserEntity $user_entity)
{ {
parent::__construct(new WebSocket($host)); parent::__construct(new WebSocket($host));
$this->_category = $category; $this->_category = $category;
$this->_user_entity = $user_entity; $this->_user_entity = $user_entity;
} }
final protected function getMyStorage() final protected function createMyStorage()
{ {
if ($this->_myStorage === null) { return new MangboardStorage($this->_category, $this->_user_entity);
$this->_myStorage = new MangboardStorage($this->_category);
}
return $this->_myStorage;
} }
protected function detail_page(int $cnt, array $listInfo): void protected function detail_page(int $cnt, array $listInfo): void
@ -49,9 +42,6 @@ class YamoonCrawler extends MyCrawler
$baord_name = "board_" . $this->_category; $baord_name = "board_" . $this->_category;
$boardsModel = new BoardsModel(); $boardsModel = new BoardsModel();
$boards_entity = $boardsModel->getEntityByID("board_" . $this->_category); $boards_entity = $boardsModel->getEntityByID("board_" . $this->_category);
if ($boards_entity === null) {
throw new \Exception("boards에서 {$baord_name} 해당정보를 찾을수 없습니다.");
}
$boardModel = new BoardModel("mb_" . $baord_name); $boardModel = new BoardModel("mb_" . $baord_name);
$board_entity = $boardModel->createByCrawler( $board_entity = $boardModel->createByCrawler(
$boards_entity, $boards_entity,
@ -60,18 +50,20 @@ class YamoonCrawler extends MyCrawler
$listInfo, $listInfo,
$storages $storages
); );
if ($board_entity === null) { foreach ($storages as $storage) {
throw new \Exception("{$baord_name} 생성에 오류가 발생했습니다."); try {
$storage->backend($boards_entity, $board_entity, $boardModel->getTable());
} catch (\Exception $e) {
log_message("notice", sprintf(
"\n---%s -> %s 게시물의 %s번째:%s 파일 등록 오류---\n%s\n--------------------------------\n",
__FUNCTION__,
$board_entity->getTitle(),
$storage->getOriginSequence(),
$storage->getOriginName(),
$e->getMessage()
));
}
} }
$fileModel = new FileModel();
$fileModel->createByCrawler(
$boards_entity,
$this->_user_entity,
$board_entity,
$boardModel->getTable(),
$storages
);
$this->create_small_ImageTrait($board_entity, $storages);
} }
public function execute(int $max_limit): void public function execute(int $max_limit): void

View File

@ -2,13 +2,30 @@
namespace App\Libraries\MyStorage; namespace App\Libraries\MyStorage;
use App\Entities\Mangboard\BoardEntity;
use App\Entities\Mangboard\BoardsEntity;
use App\Entities\Mangboard\UserEntity; use App\Entities\Mangboard\UserEntity;
use App\Models\Mangboard\FileModel;
use App\Traits\ImageTrait;
class MangboardStorage extends FileStorage class MangboardStorage extends FileStorage
{ {
public function __construct(string $path) use ImageTrait;
private $_category = "";
private $_user_entity = null;
private $_fileModel = null;
public function __construct(string $category, UserEntity $user_entity)
{ {
parent::__construct($path); parent::__construct($category);
$this->_category = $category;
$this->_user_entity = $user_entity;
}
final protected function getFileModel(): FileModel
{
if ($this->_fileModel === null) {
$this->_fileModel = new FileModel();
}
return $this->_fileModel;
} }
final public function getBasePath(): string final public function getBasePath(): string
{ {
@ -56,4 +73,16 @@ class MangboardStorage extends FileStorage
)); ));
return $content; return $content;
} }
final public function backend(BoardsEntity $boards_entity, BoardEntity $board_entity, string $board_table)
{
$this->getFileModel()->createByCrawler(
$boards_entity,
$this->_user_entity,
$board_entity,
$board_table,
$this
);
$this->create_small_ImageTrait($board_entity, $this);
}
} }

View File

@ -6,6 +6,7 @@ use App\Entities\Mangboard\BoardEntity;
use App\Entities\Mangboard\BoardsEntity; use App\Entities\Mangboard\BoardsEntity;
use App\Entities\Mangboard\FileEntity; use App\Entities\Mangboard\FileEntity;
use App\Entities\Mangboard\UserEntity; use App\Entities\Mangboard\UserEntity;
use App\Libraries\MyStorage\MangboardStorage;
use App\Models\CommonModel; use App\Models\CommonModel;
// +------------------+----------------------+------+-----+---------------------+----------------+ // +------------------+----------------------+------+-----+---------------------+----------------+
@ -129,10 +130,8 @@ class FileModel extends CommonModel
UserEntity $user_entity, UserEntity $user_entity,
BoardEntity $board_entity, BoardEntity $board_entity,
string $board_table, string $board_table,
array $storages MangboardStorage $storage
): void { ): FileEntity {
foreach ($storages as $storage) {
try {
//파일관리 table에 등록 //파일관리 table에 등록
$formDatas = []; $formDatas = [];
//Board PID 넣기 //Board PID 넣기
@ -158,16 +157,6 @@ class FileModel extends CommonModel
$storage->getOriginSequence(), $storage->getOriginSequence(),
$entity->getTitle() $entity->getTitle()
)); ));
} catch (\Exception $e) { return $entity;
log_message("notice", sprintf(
"\n---%s -> %s 게시물의 %s번째:%s 파일 등록 오류---\n%s\n--------------------------------\n",
__FUNCTION__,
$board_entity->getTitle(),
$storage->getOriginSequence(),
$storage->getOriginName(),
$e->getMessage()
));
}
}
} }
} }

View File

@ -3,6 +3,7 @@
namespace App\Traits; namespace App\Traits;
use App\Entities\Mangboard\BoardEntity; use App\Entities\Mangboard\BoardEntity;
use App\Libraries\MyStorage\MangboardStorage;
trait ImageTrait trait ImageTrait
{ {
@ -100,10 +101,8 @@ trait ImageTrait
imagedestroy($this->_image); imagedestroy($this->_image);
} }
public function create_small_ImageTrait(BoardEntity $board_entity, array $storages, $target_name = "small", int $width = 480, int $height = 319): void public function create_small_ImageTrait(BoardEntity $board_entity, MangboardStorage $storage, $target_name = "small", int $width = 480, int $height = 319): void
{ {
try {
foreach ($storages as $storage) {
$fileInfo = pathinfo($storage->getFullPath() . DIRECTORY_SEPARATOR . $storage->getOriginName(), PATHINFO_ALL); $fileInfo = pathinfo($storage->getFullPath() . DIRECTORY_SEPARATOR . $storage->getOriginName(), PATHINFO_ALL);
$target_file_name = sprintf("%s_%s.%s", $fileInfo['filename'], $target_name, $fileInfo['extension']); $target_file_name = sprintf("%s_%s.%s", $fileInfo['filename'], $target_name, $fileInfo['extension']);
if (!$this->isFileType_FileTrait($fileInfo['extension'])) { if (!$this->isFileType_FileTrait($fileInfo['extension'])) {
@ -133,15 +132,4 @@ trait ImageTrait
$target_file_name $target_file_name
)); ));
} }
} catch (\Exception $e) {
log_message("notice", sprintf(
"\n---%s -> %s 게시물의 %s번째:%s 작은이미지 생성 오류---\n%s\n--------------------------------\n",
__FUNCTION__,
$board_entity->getTitle(),
$storage->getOriginSequence(),
$storage->getOriginName(),
$e->getMessage()
));
}
}
} }