idcjp/Automation
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Automation init...3

Browse Source
This commit is contained in:
최준흠 2024-09-16 18:22:39 +09:00
parent 3341cdecd2
commit 4eb8d52318
19 changed files with 354 additions and 492 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
app/Controllers/Mangboard/CrawlerController.php
View File

@ -3,20 +3,44 @@
namespace App\Controllers\Mangboard; namespace App\Controllers\Mangboard;
use App\Controllers\CommonController; use App\Controllers\CommonController;
use App\Libraries\Mangboard\User;
use App\Entities\Mangboard\UserEntity; use App\Entities\Mangboard\UserEntity;
use App\Libraries\MyCrawler\YamapCrawler; use App\Libraries\MyCrawler\YamapCrawler;
use App\Libraries\MyCrawler\YamoonCrawler; use App\Libraries\MyCrawler\YamoonCrawler;
use App\Models\Mangboard\UserModel;
class CrawlerController extends CommonController class CrawlerController extends CommonController
{ {
private $_user = null; private $_userModel = null;
private function getUser() public function getUserModel(): UserModel
{ {
if ($this->_user === null) { if ($this->_user_model === null) {
$this->_user = new User(); return $this->_user_model = new UserModel();
} }
return $this->_user; return $this->_user_model;
}
public function login(string $host, string $id, string $password): bool|UserEntity
{
$user_entity = $this->getUserModel()->getEntityByID($id);
// $response = $this->getWebLibrary($host)->getResponse(
// $host . getenv("mangboard.login.url"),
// "post",
// [
// 'form_params' => [
// 'user_id' => $id,
// 'password' => $password,
// ],
// ]
// );
// if ($response->getStatusCode() == 200) {
// $entity = $this->getUserModel()->getEntityByLoginCheck($id, $password);
// if ($entity === null) {
// throw new \Exception("{$id}는 회원이 아니거나 암호가 맞지 않습니다.");
// }
// } else {
// throw new \Exception("연결실패:" . $response->getStatusCode());
// }
log_message("notice", "{$id}로 로그인 성공");
return $user_entity;
} }
public function yamap(string $category, string $id = "", string $debug = "false"): string public function yamap(string $category, string $id = "", string $debug = "false"): string
{ {
@ -24,7 +48,7 @@ class CrawlerController extends CommonController
$id = $id == "" ? getenv("mangboard.login.default.id") : $id; $id = $id == "" ? getenv("mangboard.login.default.id") : $id;
$password = getenv("mangboard.login.default.password"); $password = getenv("mangboard.login.default.password");
//1. 사이트 로그인 처리 //1. 사이트 로그인 처리
$user_entity = $this->getUser()->login(getenv("mangboard.host.url"), $id, $password); $user_entity = $this->login(getenv("mangboard.host.url"), $id, $password);
//2. 필요한 로그인한 사용자정보,Socket,Storage 정의후 Crawler에게 전달. //2. 필요한 로그인한 사용자정보,Socket,Storage 정의후 Crawler에게 전달.
$crawler = new YamapCrawler(getenv('yamap.host.url'), $category, $user_entity); $crawler = new YamapCrawler(getenv('yamap.host.url'), $category, $user_entity);
$crawler->setDebug($debug === "true" ? true : false); $crawler->setDebug($debug === "true" ? true : false);
@ -41,7 +65,7 @@ class CrawlerController extends CommonController
$id = $id == "" ? getenv("mangboard.login.default.id") : $id; $id = $id == "" ? getenv("mangboard.login.default.id") : $id;
$password = getenv("mangboard.login.default.password"); $password = getenv("mangboard.login.default.password");
//1. 사이트 로그인 처리 //1. 사이트 로그인 처리
$user_entity = $this->getUser()->login(getenv("mangboard.host.url"), $id, $password); $user_entity = $this->login(getenv("mangboard.host.url"), $id, $password);
//2. 필요한 로그인한 사용자정보,Socket,Storage 정의후 Crawler에게 전달. //2. 필요한 로그인한 사용자정보,Socket,Storage 정의후 Crawler에게 전달.
$crawler = new YamoonCrawler(getenv("yamoon.host.url"), $category, $user_entity); $crawler = new YamoonCrawler(getenv("yamoon.host.url"), $category, $user_entity);
$crawler->setDebug($debug === "true" ? true : false); $crawler->setDebug($debug === "true" ? true : false);

80
app/Libraries/Mangboard/Board.php
View File

@ -1,80 +0,0 @@
<?php
namespace App\Libraries\Mangboard;
use App\Libraries\CommonLibrary;
use App\Models\Mangboard\BoardModel;
use App\Entities\Mangboard\UserEntity;
use App\Entities\Mangboard\BoardsEntity;
use App\Entities\Mangboard\BoardEntity;
class Board extends CommonLibrary
{
private $_model = null;
private $_boards_entity = null;
private $_user_entity = null;
public function __construct(BoardsEntity $boards_entity, UserEntity $user_entity)
{
parent::__construct();
$this->_boards_entity = $boards_entity;
$this->_user_entity = $user_entity;
}
public function getModel(): BoardModel
{
if ($this->_model === null) {
$table_name = "mb_" . $this->_boards_entity->getTitle();
$this->_model = new BoardModel($table_name);
}
return $this->_model;
}
public function getBoardsEntity(): BoardsEntity
{
return $this->_boards_entity;
}
public function getUserEntity(): UserEntity
{
return $this->_user_entity;
}
public function createByCrawler(int $cnt, array $listInfo, array $storages): BoardEntity
{
$formDatas = [];
//미디어관련정보 entity에 넣기
$formDatas['title'] = $listInfo["title"];
$formDatas['user_pid'] = $this->getUserEntity()->getPK();
$formDatas['user_id'] = $this->getUserEntity()->getID();
$formDatas['user_name'] = $listInfo["nickname"] != "" ? $listInfo["nickname"] : $this->getUserEntity()->getTitle();
$formDatas['level'] = $this->getBoardsEntity()->getListLevel();
$formDatas['hit'] = $listInfo['hit'];
$formDatas['reg_date'] = date("Y-m-d H:i:s", strtotime($listInfo['date']));
$formDatas['data_type'] = "html";
$formDatas['editor_type'] = "S";
$formDatas['image_path'] = "";
$formDatas['content'] = "";
foreach ($storages as $storage) {
if ($formDatas['image_path'] == "") {
$formDatas['image_path'] = $storage->getBasePath() . DIRECTORY_SEPARATOR . $storage->getPath() . DIRECTORY_SEPARATOR . $storage->getOriginName();
}
$formDatas['content'] .= $storage->getHTMLTag();
}
//망보드 게시판에 등록
if ($formDatas['content'] == "") {
throw new \Exception(sprintf(
"%s=>%s번째 %s 내용이 없어 => %s 등록 취소",
__FUNCTION__,
$cnt,
$listInfo["title"],
$this->getModel()->getTable()
));
}
$entity = $this->getModel()->create($formDatas);
log_message("notice", sprintf(
"%s=>%s번째 %s => %s 등록 완료",
__FUNCTION__,
$cnt,
$listInfo["title"],
$this->getModel()->getTable()
));
return $entity;
}
}

48
app/Libraries/Mangboard/Boards.php
View File

@ -1,48 +0,0 @@
<?php
namespace App\Libraries\Mangboard;
use App\Libraries\CommonLibrary;
use App\Models\Mangboard\BoardsModel;
use App\Entities\Mangboard\UserEntity;
use App\Entities\Mangboard\BoardsEntity;
class Boards extends CommonLibrary
{
private $_model = null;
private $_entity = null;
private $_category = null;
private $_user_entity = null;
public function __construct(string $category, UserEntity $uer_entity)
{
parent::__construct();
$this->_category = $category;
$this->_user_entity = $uer_entity;
}
public function getModel(): BoardsModel
{
if ($this->_model === null) {
$this->_model = new BoardsModel();
}
return $this->_model;
}
public function getEntity(): BoardsEntity
{
if ($this->_entity === null) {
$this->_entity = $this->getModel()->getEntityByID("board_" . $this->getCategory());
}
return $this->_entity;
}
//---------------------------------------------------------------------//
public function getCategory(): string
{
return $this->_category;
}
public function getUserEntity(): UserEntity
{
if ($this->_user_entity === null) {
throw new \Exception("사용자정보가 없습니다.");
}
return $this->_user_entity;
}
}

84
app/Libraries/Mangboard/File.php
View File

@ -1,84 +0,0 @@
<?php
namespace App\Libraries\Mangboard;
use App\Libraries\CommonLibrary;
use App\Libraries\MyStorage\MangboardStorage;
use App\Models\Mangboard\FileModel;
use App\Entities\Mangboard\UserEntity;
use App\Entities\Mangboard\FileEntity;
use App\Entities\Mangboard\BoardsEntity;
use App\Entities\Mangboard\BoardEntity;
class File extends CommonLibrary
{
private $_model = null;
private $_boards_entity = null;
private $_user_entity = null;
public function __construct(BoardsEntity $boards_entity, UserEntity $user_entity)
{
parent::__construct();
$this->_boards_entity = $boards_entity;
$this->_user_entity = $user_entity;
}
public function getModel(): FileModel
{
if ($this->_model === null) {
return $this->_model = new FileModel();
}
return $this->_model;
}
public function getBoardsEntity(): BoardsEntity
{
return $this->_boards_entity;
}
public function getUserEntity(): UserEntity
{
return $this->_user_entity;
}
public function create(BoardEntity $board_entity, MangboardStorage $storage): FileEntity
{
//파일관리 table에 등록
$formDatas = [];
//Board PID 넣기
$formDatas['board_pid'] = $board_entity->getPk();
$formDatas['user_pid'] = $this->getUserEntity()->getPK();
$formDatas['user_name'] = $this->getUserEntity()->getTitle();
$formDatas['board_name'] = $this->getBoardsEntity()->getTitle();
$formDatas['table_name'] = $this->getModel()->getTable();
$formDatas['file_path'] = $storage->getBasePath() . DIRECTORY_SEPARATOR . $storage->getPath() . DIRECTORY_SEPARATOR . $storage->getOriginName();
$formDatas['file_name'] = $storage->getOriginName();
$formDatas['file_type'] = $storage->getMimeType();
$formDatas['file_caption'] = $storage->getOriginName();
$formDatas['file_alt'] = $storage->getOriginName();
$formDatas['file_description'] = "Filedata";
$formDatas['file_size'] = $storage->getFileSize();
$formDatas['file_sequence'] = $storage->getOriginSequence();
$formDatas['reg_date'] = date("Y-m-d H:i:s");
return $this->getModel()->create($formDatas);
}
public function createByCrawler(BoardEntity $board_entity, array $storages): void
{
try {
foreach ($storages as $storage) {
$entity = $this->create($board_entity, $storage);
log_message("notice", sprintf(
"%s -> %s 게시물의 %s번째:%s 파일 등록 완료",
__FUNCTION__,
$board_entity->getTitle(),
$storage->getOriginSequence(),
$entity->getTitle()
));
}
} catch (\Exception $e) {
log_message("notice", sprintf(
"\n---%s -> %s 게시물의 %s번째:%s 파일 등록 오류---\n%s\n--------------------------------\n",
__FUNCTION__,
$board_entity->getTitle(),
$storage->getOriginSequence(),
$storage->getOriginName(),
$e->getMessage()
));
}
}
}

68
app/Libraries/Mangboard/Image.php
View File

@ -1,68 +0,0 @@
<?php
namespace App\Libraries\Mangboard;
use App\Traits\ImageTrait;
use App\Traits\FileTrait;
use App\Libraries\MyStorage\MangboardStorage;
use App\Libraries\CommonLibrary;
use App\Entities\Mangboard\BoardEntity;
class Image extends CommonLibrary
{
use FileTrait, ImageTrait;
public function __construct()
{
parent::__construct();
}
public function create(MangboardStorage $storage, $target = "small", int $width = 480, int $height = 319): string
{
$fileInfo = pathinfo($storage->getFullPath() . DIRECTORY_SEPARATOR . $storage->getOriginName(), PATHINFO_ALL);
$target_file_name = sprintf("%s_%s.%s", $fileInfo['filename'], $target, $fileInfo['extension']);
if (!$this->isFileType_FileTrait($fileInfo['extension'])) {
throw new \Exception("{$storage->getOriginName()} Image 형식파일이 아닙니다.");
}
// 이미지 파일 로드
$this->load_ImageTrait($storage->getFullPath() . DIRECTORY_SEPARATOR . $storage->getOriginName());
// 200x200으로 이미지 크기 조정
$this->resize_ImageTrait($width, $height);
// 파일 저장
$this->save_ImageTrait($storage->getFullPath() . DIRECTORY_SEPARATOR . $target_file_name);
// 메모리 해제
$this->destroy_ImageTrait();
log_message("debug", sprintf(
"%s %s->%s(W:%s,H:%s) 작업완료)",
__FUNCTION__,
$storage->getOriginName(),
$target_file_name,
$width,
$height
));
return $target_file_name;
}
public function createByCrawler(BoardEntity $board_entity, array $storages): void
{
try {
foreach ($storages as $storage) {
$file_name = $this->create($storage);
log_message("notice", sprintf(
"%s -> %s 게시물의 %s번째:%s 작은이미지 생성 완료",
__FUNCTION__,
$board_entity->getTitle(),
$storage->getOriginSequence(),
$file_name
));
}
} catch (\Exception $e) {
log_message("notice", sprintf(
"\n---%s -> %s 게시물의 %s번째:%s 작은이미지 생성 오류---\n%s\n--------------------------------\n",
__FUNCTION__,
$board_entity->getTitle(),
$storage->getOriginSequence(),
$storage->getOriginName(),
$e->getMessage()
));
}
}
}

48
app/Libraries/Mangboard/User.php
View File

@ -1,48 +0,0 @@
<?php
namespace App\Libraries\Mangboard;
use App\Models\Mangboard\UserModel;
use App\Entities\Mangboard\UserEntity;
use App\Libraries\CommonLibrary;
class User extends CommonLibrary
{
private $_user_model = null;
public function __construct()
{
parent::__construct();
}
public function getUserModel(): UserModel
{
if ($this->_user_model === null) {
return $this->_user_model = new UserModel();
}
return $this->_user_model;
}
// 로그인 메서드
public function login(string $host, string $id, string $password): bool|UserEntity
{
$entity = $this->getUserModel()->getEntityByID($id);
// $response = $this->getWebLibrary($host)->getResponse(
// $host . getenv("mangboard.login.url"),
// "post",
// [
// 'form_params' => [
// 'user_id' => $id,
// 'password' => $password,
// ],
// ]
// );
// if ($response->getStatusCode() == 200) {
// $entity = $this->getUserModel()->getEntityByLoginCheck($id, $password);
// if ($entity === null) {
// throw new \Exception("{$id}는 회원이 아니거나 암호가 맞지 않습니다.");
// }
// } else {
// throw new \Exception("연결실패:" . $response->getStatusCode());
// }
log_message("notice", "{$id}로 로그인 성공");
return $entity;
}
}

0
app/Libraries/Cloudflare/Account.php → app/Libraries/MyCloudflare/Account.php
View File

0
app/Libraries/CloudflareLibrary.php → app/Libraries/MyCloudflare/CloudflareLibrary.php
View File

0
app/Libraries/Cloudflare/Firewall.php → app/Libraries/MyCloudflare/Firewall.php
View File

0
app/Libraries/Cloudflare/Magictransit/AllowList.php → app/Libraries/MyCloudflare/Magictransit/AllowList.php
View File

0
app/Libraries/Cloudflare/Record.php → app/Libraries/MyCloudflare/Record.php
View File

0
app/Libraries/Cloudflare/Zone.php → app/Libraries/MyCloudflare/Zone.php
View File

30
app/Libraries/MyCrawler/MyCrawler.php
View File

@ -10,16 +10,13 @@ abstract class MyCrawler extends CommonLibrary
{ {
use FileTrait; use FileTrait;
private $_mySocket = null; private $_mySocket = null;
protected $_storages = [];
protected function __construct($mySocket) protected function __construct($mySocket)
{ {
parent::__construct(); parent::__construct();
$this->_mySocket = $mySocket; $this->_mySocket = $mySocket;
} }
abstract protected function getMyStorage(); abstract protected function getMyStorage();
abstract protected function list_page(): array; abstract protected function detail_page(int $cnt, array $listInfo): void;
abstract protected function detail_page(array $listInfo): array;
abstract protected function backend_process(int $i, array $listInfo, array $storages);
final protected function getMySocket() final protected function getMySocket()
{ {
if ($this->_mySocket === null) { if ($this->_mySocket === null) {
@ -70,14 +67,14 @@ abstract class MyCrawler extends CommonLibrary
} }
//--------미디어 관련------- //--------미디어 관련-------
private function media_save(int $file_sequence, string $media_type, string $file_name, string $content): void private function media_save(int $file_sequence, string $media_type, string $file_name, string $content): mixed
{ {
log_message("debug", __FUNCTION__ . " 원본파일 {$file_name} 작업 시작"); log_message("debug", __FUNCTION__ . " 원본파일 {$file_name} 작업 시작");
$this->getMyStorage()->setOriginName($file_name); $this->getMyStorage()->setOriginName($file_name);
$this->getMyStorage()->setOriginContent($content); $this->getMyStorage()->setOriginContent($content);
$this->getMyStorage()->setOriginType($media_type); $this->getMyStorage()->setOriginType($media_type);
$this->getMyStorage()->setOriginSequence($file_sequence); $this->getMyStorage()->setOriginSequence($file_sequence);
$this->_storages[] = $this->getMyStorage()->save(); return $this->getMyStorage()->save();
} }
//Yamap ViewPage의 이미지나영상데이터가 있으면 Dodownload 한다. //Yamap ViewPage의 이미지나영상데이터가 있으면 Dodownload 한다.
private function media_download(string $media_type, string $url): array private function media_download(string $media_type, string $url): array
@ -98,19 +95,17 @@ abstract class MyCrawler extends CommonLibrary
} }
final protected function media_process(array $media_urls): array final protected function media_process(array $media_urls): array
{ {
log_message("debug", var_export($media_urls, true));
$file_sequence = 1; $file_sequence = 1;
$this->_storages = []; //CreateBoard에서 사용을 위해 DetailPage마다 초기화 $storages = []; //CreateBoard에서 사용을 위해 DetailPage마다 초기화
// log_message("debug", var_export($urls, true)); // log_message("debug", var_export($urls, true));
foreach ($media_urls as $media_type => $urls) { foreach ($media_urls as $media_type => $urls) {
foreach ($urls as $url) { foreach ($urls as $url) {
try { try {
if ($url === null) {
continue;
}
list($file_name, $content) = $this->media_download($media_type, $url); list($file_name, $content) = $this->media_download($media_type, $url);
$this->media_save($file_sequence, $media_type, $file_name, $content); $storages[] = $this->media_save($file_sequence, $media_type, $file_name, $content);
$file_sequence++; $file_sequence++;
log_message("notice", __FUNCTION__ . " OriginType->{$media_type} 작업 완료"); log_message("notice", __FUNCTION__ . " MediaType->{$media_type} 작업 완료");
} catch (\Exception $e) { } catch (\Exception $e) {
log_message("warning", sprintf( log_message("warning", sprintf(
"\n---%s mediaType->%s 오류---\n%s\n-----------------------------------------\n", "\n---%s mediaType->%s 오류---\n%s\n-----------------------------------------\n",
@ -121,10 +116,7 @@ abstract class MyCrawler extends CommonLibrary
} }
} }
} }
if (!count($this->_storages)) { return $storages;
throw new \Exception("Download된 Content가 없습니다.");
}
return $this->_storages;
} }
protected function main_process(int $max_limit, array $listInfos): void protected function main_process(int $max_limit, array $listInfos): void
{ {
@ -141,11 +133,7 @@ abstract class MyCrawler extends CommonLibrary
try { try {
log_message("notice", "게시물 {$i}번째/{$total}개중 {$listInfo["nickname"]} 작업시작"); log_message("notice", "게시물 {$i}번째/{$total}개중 {$listInfo["nickname"]} 작업시작");
//listInfo는 title,작성자,작성시간등등의 정보를 가지고 있어 detail_page 처리 안에서 바뀔 수 있으므로 다시 반환 받는다. //listInfo는 title,작성자,작성시간등등의 정보를 가지고 있어 detail_page 처리 안에서 바뀔 수 있으므로 다시 반환 받는다.
list($listInfo, $media_urls) = $this->detail_page($listInfo); $this->detail_page($i, $listInfo);
//Image 나 Video 소스들의 url을 가져와서 실제 다운받는 처리
$this->media_process($media_urls);
//File DB 및 Board DB 등록작업등
$this->backend_process($i, $listInfo, $this->_storages);
log_message("notice", "게시물 {$i}번째/{$total}개중 {$listInfo["nickname"]} 작업완료."); log_message("notice", "게시물 {$i}번째/{$total}개중 {$listInfo["nickname"]} 작업완료.");
$i++; $i++;
} catch (\Exception $e) { } catch (\Exception $e) {

127
app/Libraries/MyCrawler/YamapCrawler.php
View File

@ -2,13 +2,18 @@
namespace App\Libraries\MyCrawler; namespace App\Libraries\MyCrawler;
use App\Entities\Mangboard\UserEntity;
use App\Libraries\MySocket\WebSocket; use App\Libraries\MySocket\WebSocket;
use App\Libraries\MyStorage\MangboardStorage; use App\Libraries\MyStorage\MangboardStorage;
use App\Entities\Mangboard\UserEntity; use App\Models\Mangboard\BoardModel;
use App\Models\Mangboard\BoardsModel;
use App\Models\Mangboard\FileModel;
use App\Traits\ImageTrait;
use Symfony\Component\DomCrawler\Crawler; use Symfony\Component\DomCrawler\Crawler;
class YamapCrawler extends MyCrawler class YamapCrawler extends MyCrawler
{ {
use ImageTrait;
private $_category = ""; private $_category = "";
private $_user_entity = null; private $_user_entity = null;
private $_myStorage = null; private $_myStorage = null;
@ -55,60 +60,88 @@ class YamapCrawler extends MyCrawler
// </div> // </div>
// <div id="freesubframe"></div> // <div id="freesubframe"></div>
// </div> // </div>
protected function detail_page(array $listInfo): array protected function detail_page(int $cnt, array $listInfo): void
{ {
$response = $this->getMySocket()->getContent($listInfo['detail_url']); $response = $this->getMySocket()->getContent($listInfo['detail_url']);
$tag = getenv("yamap.view.content.tag"); $tag = getenv("yamap.view.content.tag");
return $this->getMediaUrls($response, $tag, $listInfo); list($listInfo, $media_urls) = $this->getMediaUrls($response, $tag, $listInfo);
} //Image 나 Video 소스들의 url을 가져와서 실제 다운받는 처리
protected function list_page(): array $storages = $this->media_process($media_urls);
{ if (!count($storages)) {
if ($this->getDebug()) { throw new \Exception("등록할 자료가 없습니다.");
return [
'title' => getenv("yamap.view.test.title"),
'nickname' => getenv("yamap.view.test.nickname"),
'detail_url' => getenv("yamap.view.test.url"),
'time' => date("Y-m-d H:i:s"),
'hit' => 1,
];
} }
$listInfos = []; //File DB 및 Board DB 등록작업등
$response = $this->getMySocket()->getContent(getenv("yamap.list.url.{$this->_category}")); $baord_name = "board_" . $this->_category;
$selector = $this->getSelector($response, getenv("yamap.list.tag")); $boardsModel = new BoardsModel();
//div.bbs_item를 가진 객체를 찾아서 같은 형식의 객체(sibling)를 배열로 넘김 $boards_entity = $boardsModel->getEntityByID("board_" . $this->_category);
// log_message("debug", sprintf("\n-------------MainPage------------\n%s\n--------------------------\n", $selector->html())); if ($boards_entity === null) {
$selector->filter(getenv("yamap.list.item.tag"))->each( throw new \Exception("boards에서 {$baord_name} 해당정보를 찾을수 없습니다.");
function (Crawler $node) use (&$listInfos): void { }
//bbs_item에서 span.g_nickname 객체를 찾아서 작성자가 "관리자" 아닌지 확인 후 Return Bool $boardModel = new BoardModel("mb_" . $baord_name);
$nickname = $node->filter(getenv("yamap.list.item.nickname.tag"))->text(); $board_entity = $boardModel->createByCrawler(
$hit = $node->filter(getenv("yamap.list.item.hit.tag"))->text(); $boards_entity,
$date = $node->filter(getenv("yamap.list.item.date.tag"))->text(); $this->_user_entity,
if ($nickname != getenv("yamap.list.item.nickname.except")) { $cnt,
//작성자가 "관리자"가 아니 게시물이면 해당 bbs_item에서 a.list_subject 객체를 찾아서 $listInfo,
$link_node = $node->filter(getenv("yamap.list.item.link.tag")); $storages
$detail_url = $link_node->attr("href");
$title = $link_node->children()->last()->text();
$listInfos[] = ['title' => $title, 'nickname' => $nickname, 'detail_url' => $detail_url, 'date' => $date, 'hit' => $hit];
}
}
); );
if (!count($listInfos)) { if ($board_entity === null) {
throw new \Exception("Target URL이 없습니다."); throw new \Exception("{$baord_name} 생성에 오류가 발생했습니다.");
} }
log_message("notice", __FUNCTION__ . " 작업 완료"); $fileModel = new FileModel();
return $listInfos; $fileModel->createByCrawler(
} $boards_entity,
protected function backend_process(int $i, array $listInfo, array $storages) $this->_user_entity,
{ $board_entity,
//File DB 및 Board DB 등록작업 $boardModel->getTable(),
$board_entity = $this->getMyStorage()->getBoard()->createByCrawler($i, $listInfo, $storages); $storages
$this->getMyStorage()->getFile()->createByCrawler($board_entity, $storages); );
$this->getMyStorage()->getImage()->createByCrawler($board_entity, $storages); $this->create_small_ImageTrait($board_entity, $storages);
} }
public function execute(int $max_limit): void public function execute(int $max_limit): void
{ {
$listInfos = $this->list_page(); try {
$this->main_process($max_limit, $listInfos); $listInfos = [];
log_message("notice", __FUNCTION__ . " 작업이 완료되었습니다."); if ($this->getDebug()) {
$listInfos = [
'title' => getenv("yamap.view.test.title"),
'nickname' => getenv("yamap.view.test.nickname"),
'detail_url' => getenv("yamap.view.test.url"),
'time' => date("Y-m-d H:i:s"),
'hit' => 1,
];
} else {
$response = $this->getMySocket()->getContent(getenv("yamap.list.url.{$this->_category}"));
$selector = $this->getSelector($response, getenv("yamap.list.tag"));
//div.bbs_item를 가진 객체를 찾아서 같은 형식의 객체(sibling)를 배열로 넘김
// log_message("debug", sprintf("\n-------------MainPage------------\n%s\n--------------------------\n", $selector->html()));
$selector->filter(getenv("yamap.list.item.tag"))->each(
function (Crawler $node) use (&$listInfos): void {
//bbs_item에서 span.g_nickname 객체를 찾아서 작성자가 "관리자" 아닌지 확인 후 Return Bool
$nickname = $node->filter(getenv("yamap.list.item.nickname.tag"))->text();
$hit = $node->filter(getenv("yamap.list.item.hit.tag"))->text();
$date = $node->filter(getenv("yamap.list.item.date.tag"))->text();
if ($nickname != getenv("yamap.list.item.nickname.except")) {
//작성자가 "관리자"가 아니 게시물이면 해당 bbs_item에서 a.list_subject 객체를 찾아서
$link_node = $node->filter(getenv("yamap.list.item.link.tag"));
$detail_url = $link_node->attr("href");
$title = $link_node->children()->last()->text();
$listInfos[] = ['title' => $title, 'nickname' => $nickname, 'detail_url' => $detail_url, 'date' => $date, 'hit' => $hit];
}
}
);
}
if (!count($listInfos)) {
throw new \Exception("Target URL이 없습니다.");
}
$this->main_process($max_limit, $listInfos);
log_message("notice", __FUNCTION__ . " 작업이 완료되었습니다.");
} catch (\Exception $e) {
log_message("warning", sprintf(
"\n---%s 오류---\n%s\n-----------------------------------------\n",
__FUNCTION__,
$e->getMessage()
));
}
} }
} }

136
app/Libraries/MyCrawler/YamoonCrawler.php
View File

@ -2,13 +2,18 @@
namespace App\Libraries\MyCrawler; namespace App\Libraries\MyCrawler;
use App\Entities\Mangboard\UserEntity;
use App\Libraries\MySocket\WebSocket; use App\Libraries\MySocket\WebSocket;
use App\Libraries\MyStorage\MangboardStorage; use App\Libraries\MyStorage\MangboardStorage;
use App\Entities\Mangboard\UserEntity; use App\Models\Mangboard\BoardModel;
use App\Models\Mangboard\BoardsModel;
use App\Models\Mangboard\FileModel;
use App\Traits\ImageTrait;
use Symfony\Component\DomCrawler\Crawler; use Symfony\Component\DomCrawler\Crawler;
class YamoonCrawler extends MyCrawler class YamoonCrawler extends MyCrawler
{ {
use ImageTrait;
private $_category = ""; private $_category = "";
private $_user_entity = null; private $_user_entity = null;
private $_myStorage = null; private $_myStorage = null;
@ -25,7 +30,8 @@ class YamoonCrawler extends MyCrawler
} }
return $this->_myStorage; return $this->_myStorage;
} }
protected function detail_page(array $listInfo): array
protected function detail_page(int $cnt, array $listInfo): void
{ {
$response = $this->getMySocket()->getContent("/newboard/yamoonboard/" . $listInfo['detail_url']); $response = $this->getMySocket()->getContent("/newboard/yamoonboard/" . $listInfo['detail_url']);
//작성시간 //작성시간
@ -33,60 +39,88 @@ class YamoonCrawler extends MyCrawler
// $listInfo['date'] = trim($selector->text()); // $listInfo['date'] = trim($selector->text());
//작성내용 //작성내용
$tag = getenv("yamoon.view.content.tag"); $tag = getenv("yamoon.view.content.tag");
return $this->getMediaUrls($response, $tag, $listInfo); list($listInfo, $media_urls) = $this->getMediaUrls($response, $tag, $listInfo);
} //Image 나 Video 소스들의 url을 가져와서 실제 다운받는 처리
protected function list_page(): array $storages = $this->media_process($media_urls);
{ if (!count($storages)) {
if ($this->getDebug()) { throw new \Exception("등록할 자료가 없습니다.");
$listInfos = [
'title' => getenv("yamoon.view.test.title"),
'nickname' => getenv("yamoon.view.test.nickname"),
'detail_url' => getenv("yamoon.view.test.url"),
'time' => date("Y-m-d H:i:s"),
'hit' => 1,
];
} else {
} }
$listInfos = []; //File DB 및 Board DB 등록작업등
$response = $this->getMySocket()->getContent(getenv("yamoon.list.url.{$this->_category}")); $baord_name = "board_" . $this->_category;
//div.bbs_item를 가진 객체를 찾아서 같은 형식의 객체(sibling)를 배열로 넘김 $boardsModel = new BoardsModel();
// log_message("debug", sprintf("\n-------------MainPage------------\n%s\n--------------------------\n", $selector->html())); $boards_entity = $boardsModel->getEntityByID("board_" . $this->_category);
// <td class="listvisited mobile-td subject-view"> if ($boards_entity === null) {
// <a href="board-read.asp?fullboardname=yamoonfreeboard&mtablename=humor&num=89372&ref=85575&page=1" class="ya-tooltip mobile-bold mobile-height" title="<p><br><br><video autoplay=&quot;autoplay&quot; loop=&quot;loop&quot; muted=&quot;&quot; controls=&quot;controls&quot; width=&quot;560&quot;&quot; height=&quot; &quot;> <source src=&quot; https://files.bepick.net/bbs/2024/09/c2a20ab5771cbb934940551859fce1c8_769966583.mp4 &quot;> </video><br><br><br></p"> throw new \Exception("boards에서 {$baord_name} 해당정보를 찾을수 없습니다.");
// 졸고 있는 여군</a> }
// <i class="fa fa-commenting-o" aria-hidden="true"></i> <span class="color-red small">6</span> $boardModel = new BoardModel("mb_" . $baord_name);
// <span class="visible-xs visible-sm small"><i class="fa fa-user-o" aria-hidden="true"></i> yeeyuu | <i class="fa fa-thumbs-o-up" aria-hidden="true"></i> 6 | <i class="fa fa-eye" aria-hidden="true"></i> 369 | No 89372 | 2024-09-13</span> $board_entity = $boardModel->createByCrawler(
// </td> $boards_entity,
//bbs_item에서 span.g_nickname 객체를 찾아서 작성자가 "관리자" 아닌지 확인 후 Return Bool $this->_user_entity,
$this->getSelector($response, getenv("yamoon.list.tag"))->each( $cnt,
function (Crawler $node) use (&$listInfos): void { $listInfo,
$link_node = $node->filter(getenv("yamoon.list.item.link.tag")); $storages
$detail_url = $link_node->attr("href");
$title = $link_node->text();
$info_node = $node->filter(getenv("yamoon.list.item.info.tag"));
$infos = explode("|", $info_node->text());
if (trim($infos[4]) == date("Y-m-d")) {
$listInfos[] = ['title' => $title, 'detail_url' => $detail_url, 'nickname' => trim($infos[0]), 'hit' => trim($infos[2]), 'date' => trim($infos[4])];
}
}
); );
if (!count($listInfos)) { if ($board_entity === null) {
throw new \Exception("Target URL이 없습니다."); throw new \Exception("{$baord_name} 생성에 오류가 발생했습니다.");
} }
log_message("notice", __FUNCTION__ . " 작업 완료"); $fileModel = new FileModel();
return $listInfos; $fileModel->createByCrawler(
} $boards_entity,
//File DB 및 Board DB 등록작업등 $this->_user_entity,
protected function backend_process(int $i, array $listInfo, array $storages) $board_entity,
{ $boardModel->getTable(),
$board_entity = $this->getMyStorage()->getBoard()->createByCrawler($i, $listInfo, $storages); $storages
$this->getMyStorage()->getFile()->createByCrawler($board_entity, $storages); );
$this->getMyStorage()->getImage()->createByCrawler($board_entity, $storages); $this->create_small_ImageTrait($board_entity, $storages);
} }
public function execute(int $max_limit): void public function execute(int $max_limit): void
{ {
$listInfos = $this->list_page(); try {
$this->main_process($max_limit, $listInfos); $listInfos = [];
log_message("notice", __FUNCTION__ . " 작업이 완료되었습니다."); if ($this->getDebug()) {
$listInfos = [
'title' => getenv("yamoon.view.test.title"),
'nickname' => getenv("yamoon.view.test.nickname"),
'detail_url' => getenv("yamoon.view.test.url"),
'time' => date("Y-m-d H:i:s"),
'hit' => 1,
];
} else {
$response = $this->getMySocket()->getContent(getenv("yamoon.list.url.{$this->_category}"));
//div.bbs_item를 가진 객체를 찾아서 같은 형식의 객체(sibling)를 배열로 넘김
// log_message("debug", sprintf("\n-------------MainPage------------\n%s\n--------------------------\n", $selector->html()));
// <td class="listvisited mobile-td subject-view">
// <a href="board-read.asp?fullboardname=yamoonfreeboard&mtablename=humor&num=89372&ref=85575&page=1" class="ya-tooltip mobile-bold mobile-height" title="<p><br><br><video autoplay=&quot;autoplay&quot; loop=&quot;loop&quot; muted=&quot;&quot; controls=&quot;controls&quot; width=&quot;560&quot;&quot; height=&quot; &quot;> <source src=&quot; https://files.bepick.net/bbs/2024/09/c2a20ab5771cbb934940551859fce1c8_769966583.mp4 &quot;> </video><br><br><br></p">
// 졸고 있는 여군</a>
// <i class="fa fa-commenting-o" aria-hidden="true"></i> <span class="color-red small">6</span>
// <span class="visible-xs visible-sm small"><i class="fa fa-user-o" aria-hidden="true"></i> yeeyuu | <i class="fa fa-thumbs-o-up" aria-hidden="true"></i> 6 | <i class="fa fa-eye" aria-hidden="true"></i> 369 | No 89372 | 2024-09-13</span>
// </td>
//bbs_item에서 span.g_nickname 객체를 찾아서 작성자가 "관리자" 아닌지 확인 후 Return Bool
$this->getSelector($response, getenv("yamoon.list.tag"))->each(
function (Crawler $node) use (&$listInfos): void {
$link_node = $node->filter(getenv("yamoon.list.item.link.tag"));
$detail_url = $link_node->attr("href");
$title = $link_node->text();
$info_node = $node->filter(getenv("yamoon.list.item.info.tag"));
$infos = explode("|", $info_node->text());
if (trim($infos[4]) == date("Y-m-d")) {
$listInfos[] = ['title' => $title, 'detail_url' => $detail_url, 'nickname' => trim($infos[0]), 'hit' => trim($infos[2]), 'date' => trim($infos[4])];
}
}
);
}
if (!count($listInfos)) {
throw new \Exception("Target URL이 없습니다.");
}
$this->main_process($max_limit, $listInfos);
log_message("notice", __FUNCTION__ . " 작업이 완료되었습니다.");
} catch (\Exception $e) {
log_message("warning", sprintf(
"\n---%s 오류---\n%s\n-----------------------------------------\n",
__FUNCTION__,
$e->getMessage()
));
}
} }
} }

35
app/Libraries/MyStorage/MangboardStorage.php
View File

@ -82,39 +82,4 @@ class MangboardStorage extends FileStorage
)); ));
return $content; return $content;
} }
private function getBoards(): Boards
{
if ($this->_boards === null) {
$this->_boards = new Boards($this->getCategory(), $this->getUserEntity());
}
return $this->_boards;
}
final public function getBoard(): Board
{
if ($this->_board === null) {
$this->_board = new Board(
$this->getBoards()->getEntity(),
$this->getUserEntity()
);
}
return $this->_board;
}
final public function getFile(): File
{
if ($this->_file === null) {
$this->_file = new File(
$this->getBoards()->getEntity(),
$this->getUserEntity()
);
}
return $this->_file;
}
final public function getImage(): Image
{
if ($this->_image === null) {
$this->_image = new Image();
}
return $this->_image;
}
} }

52
app/Models/Mangboard/BoardModel.php
View File

@ -2,8 +2,10 @@
namespace App\Models\Mangboard; namespace App\Models\Mangboard;
use App\Models\CommonModel;
use App\Entities\Mangboard\BoardEntity; use App\Entities\Mangboard\BoardEntity;
use App\Entities\Mangboard\BoardsEntity;
use App\Entities\Mangboard\UserEntity;
use App\Models\CommonModel;
// +-----------------+---------------------+------+-----+---------------------+----------------+ // +-----------------+---------------------+------+-----+---------------------+----------------+
// | Field | Type | Null | Key | Default | Extra | // | Field | Type | Null | Key | Default | Extra |
@ -147,4 +149,52 @@ class BoardModel extends CommonModel
{ {
return $this->modify_process($entity, $formDatas); return $this->modify_process($entity, $formDatas);
} }
public function createByCrawler(
BoardsEntity $boards_entity,
UserEntity $user_entity,
int $cnt,
array $listInfo,
array $storages
): BoardEntity {
$formDatas = [];
//미디어관련정보 entity에 넣기
$formDatas['title'] = $listInfo["title"];
$formDatas['user_pid'] = $user_entity->getPK();
$formDatas['user_id'] = $user_entity->getID();
$formDatas['user_name'] = $listInfo["nickname"] != "" ? $listInfo["nickname"] : $user_entity->getTitle();
$formDatas['level'] = $boards_entity->getListLevel();
$formDatas['hit'] = $listInfo['hit'];
$formDatas['reg_date'] = date("Y-m-d H:i:s", strtotime($listInfo['date']));
$formDatas['data_type'] = "html";
$formDatas['editor_type'] = "S";
$formDatas['image_path'] = "";
$formDatas['content'] = "";
foreach ($storages as $storage) {
if ($formDatas['image_path'] == "") {
$formDatas['image_path'] = $storage->getBasePath() . DIRECTORY_SEPARATOR . $storage->getPath() . DIRECTORY_SEPARATOR . $storage->getOriginName();
}
$formDatas['content'] .= $storage->getHTMLTag();;
}
//망보드 게시판에 등록
if ($formDatas['content'] == "") {
throw new \Exception(sprintf(
"%s=>%s번째 %s 내용이 없어 => %s 등록 안함 : storage->%s",
__FUNCTION__,
$cnt,
$listInfo["title"],
$this->getTable(),
count($storages)
));
}
$entity = $this->create($formDatas);
log_message("notice", sprintf(
"%s=>%s번째 %s => %s 등록 완료 : storage->%s",
__FUNCTION__,
$cnt,
$listInfo["title"],
$this->getTable(),
count($storages)
));
return $entity;
}
} }

51
app/Models/Mangboard/FileModel.php
View File

@ -2,8 +2,11 @@
namespace App\Models\Mangboard; namespace App\Models\Mangboard;
use App\Models\CommonModel; use App\Entities\Mangboard\BoardEntity;
use App\Entities\Mangboard\BoardsEntity;
use App\Entities\Mangboard\FileEntity; use App\Entities\Mangboard\FileEntity;
use App\Entities\Mangboard\UserEntity;
use App\Models\CommonModel;
// +------------------+----------------------+------+-----+---------------------+----------------+ // +------------------+----------------------+------+-----+---------------------+----------------+
// | Field | Type | Null | Key | Default | Extra | // | Field | Type | Null | Key | Default | Extra |
@ -121,4 +124,50 @@ class FileModel extends CommonModel
{ {
return $this->modify_process($entity, $formDatas); return $this->modify_process($entity, $formDatas);
} }
public function createByCrawler(
BoardsEntity $boards_entity,
UserEntity $user_entity,
BoardEntity $board_entity,
string $board_table,
array $storages
): void {
foreach ($storages as $storage) {
try {
//파일관리 table에 등록
$formDatas = [];
//Board PID 넣기
$formDatas['board_pid'] = $board_entity->getPk();
$formDatas['user_pid'] = $user_entity->getPK();
$formDatas['user_name'] = $user_entity->getTitle();
$formDatas['board_name'] = $boards_entity->getTitle();
$formDatas['table_name'] = $board_table;
$formDatas['file_path'] = $storage->getBasePath() . DIRECTORY_SEPARATOR . $storage->getPath() . DIRECTORY_SEPARATOR . $storage->getOriginName();
$formDatas['file_name'] = $storage->getOriginName();
$formDatas['file_type'] = $storage->getMimeType();
$formDatas['file_caption'] = $storage->getOriginName();
$formDatas['file_alt'] = $storage->getOriginName();
$formDatas['file_description'] = "Filedata";
$formDatas['file_size'] = $storage->getFileSize();
$formDatas['file_sequence'] = $storage->getOriginSequence();
$formDatas['reg_date'] = date("Y-m-d H:i:s");
$entity = $this->create($formDatas);
log_message("notice", sprintf(
"%s -> %s 게시물의 %s번째:%s 파일 등록 완료",
__FUNCTION__,
$board_entity->getTitle(),
$storage->getOriginSequence(),
$entity->getTitle()
));
} catch (\Exception $e) {
log_message("notice", sprintf(
"\n---%s -> %s 게시물의 %s번째:%s 파일 등록 오류---\n%s\n--------------------------------\n",
__FUNCTION__,
$board_entity->getTitle(),
$storage->getOriginSequence(),
$storage->getOriginName(),
$e->getMessage()
));
}
}
}
} }

47
app/Traits/ImageTrait.php
View File

@ -2,6 +2,8 @@
namespace App\Traits; namespace App\Traits;
use App\Entities\Mangboard\BoardEntity;
trait ImageTrait trait ImageTrait
{ {
private $_image; private $_image;
@ -97,4 +99,49 @@ trait ImageTrait
{ {
imagedestroy($this->_image); imagedestroy($this->_image);
} }
public function create_small_ImageTrait(BoardEntity $board_entity, array $storages, $target_name = "small", int $width = 480, int $height = 319): void
{
try {
foreach ($storages as $storage) {
$fileInfo = pathinfo($storage->getFullPath() . DIRECTORY_SEPARATOR . $storage->getOriginName(), PATHINFO_ALL);
$target_file_name = sprintf("%s_%s.%s", $fileInfo['filename'], $target_name, $fileInfo['extension']);
if (!$this->isFileType_FileTrait($fileInfo['extension'])) {
throw new \Exception("{$storage->getOriginName()} Image 형식파일이 아닙니다.");
}
// 이미지 파일 로드
$this->load_ImageTrait($storage->getFullPath() . DIRECTORY_SEPARATOR . $storage->getOriginName());
// 200x200으로 이미지 크기 조정
$this->resize_ImageTrait($width, $height);
// 파일 저장
$this->save_ImageTrait($storage->getFullPath() . DIRECTORY_SEPARATOR . $target_file_name);
// 메모리 해제
$this->destroy_ImageTrait();
log_message("debug", sprintf(
"%s %s->%s(W:%s,H:%s) 작업완료)",
__FUNCTION__,
$storage->getOriginName(),
$target_file_name,
$width,
$height
));
log_message("notice", sprintf(
"%s -> %s 게시물의 %s번째:%s 작은이미지 생성 완료",
__FUNCTION__,
$board_entity->getTitle(),
$storage->getOriginSequence(),
$target_file_name
));
}
} catch (\Exception $e) {
log_message("notice", sprintf(
"\n---%s -> %s 게시물의 %s번째:%s 작은이미지 생성 오류---\n%s\n--------------------------------\n",
__FUNCTION__,
$board_entity->getTitle(),
$storage->getOriginSequence(),
$storage->getOriginName(),
$e->getMessage()
));
}
}
} }