idcjp/Automation
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Automation init...3

Browse Source
This commit is contained in:
최준흠 2024-09-18 17:19:48 +09:00
parent bceec3122e
commit 80b3e4bbc0
13 changed files with 92 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
app/Controllers/CommonController.php
View File

@ -6,18 +6,18 @@ use App\Controllers\BaseController;
abstract class CommonController extends BaseController
{
private $_datas = [];
private $_options = [];
final public function __get($name): array|null
{
if (!array_key_exists($name, $this->_datas)) {
if (!array_key_exists($name, $this->_options)) {
return null;
}
return $this->_datas;
return $this->_options;
}
final public function __set($name, $value): void
{
$this->_datas[$name] = $value;
$this->_options[$name] = $value;
}
}

8
app/Controllers/Mangboard/CrawlerController.php
View File

@ -54,7 +54,7 @@ class CrawlerController extends CommonController
$user_entity = $this->login(in_array('id', $params) ? $params['id'] : "");
//2. 필요한 로그인한 사용자정보,Socket,Storage 정의후 Crawler에게 전달.
$crawler = new YamapCrawler(getenv('yamap.host.url'), $board_name, $user_entity);
$crawler->setDebug(in_array('debug', $params));
$crawler->isDebug = in_array('debug', $params);
$crawler->execute(intval(getenv("yamap.list.max_limit")));
return "완료되었습니다.";
} catch (\Exception $e) {
@ -69,7 +69,7 @@ class CrawlerController extends CommonController
$user_entity = $this->login(in_array('id', $params) ? $params['id'] : "");
//2. 필요한 로그인한 사용자정보,Socket,Storage 정의후 Crawler에게 전달.
$crawler = new YamoonCrawler(getenv("yamoon.host.url"), $board_name, $user_entity);
$crawler->setDebug(in_array('debug', $params));
$crawler->isDebug = in_array('debug', $params);
$crawler->execute(intval(getenv("yamoon.list.max_limit")));
return "완료되었습니다.";
} catch (\Exception $e) {
@ -84,7 +84,7 @@ class CrawlerController extends CommonController
$user_entity = $this->login(in_array('id', $params) ? $params['id'] : "");
//2. 필요한 로그인한 사용자정보,Socket,Storage 정의후 Crawler에게 전달.
$crawler = new SirCrawler(getenv("sir.host.url"), $board_name, $user_entity);
$crawler->setDebug(in_array('debug', $params));
$crawler->isDebug = in_array('debug', $params);
$crawler->execute(intval(getenv("sir.list.max_limit")));
return "완료되었습니다.";
} catch (\Exception $e) {
@ -99,7 +99,7 @@ class CrawlerController extends CommonController
$user_entity = $this->login(in_array('id', $params) ? $params['id'] : "");
//2. 필요한 로그인한 사용자정보,Socket,Storage 정의후 Crawler에게 전달.
$crawler = new InvenCrawler(getenv("inven.host.url"), $board_name, $user_entity);
$crawler->setDebug(in_array('debug', $params));
$crawler->isDebug = in_array('debug', $params);
$crawler->execute(intval(getenv("inven.list.max_limit")));
return "완료되었습니다.";
} catch (\Exception $e) {

17
app/Libraries/CommonLibrary.php
View File

@ -4,15 +4,18 @@ namespace App\Libraries;
abstract class CommonLibrary
{
private $_debug = false;
private $_options = [];
protected function __construct() {}
final public function __get($name): array|null
{
if (!array_key_exists($name, $this->_options)) {
return null;
}
return $this->_options;
}
final public function getDebug(): bool
final public function __set($name, $value): void
{
return $this->_debug;
}
final public function setDebug(bool $debug): void
{
$this->_debug = $debug;
$this->_options[$name] = $value;
}
}

15
app/Libraries/MyCrawler/Mangboard/InvenCrawler.php
View File

@ -46,14 +46,13 @@ class InvenCrawler extends MangboardCrawler
// </div>
// <!-- ============== End CONTENT ============== -->
// </div>
protected function detail_page(int $cnt, array $listInfo): array
protected function detail_process(int $cnt, array $listInfo): array
{
$response = $this->getMySocket()->getContent($listInfo['detail_url']);
$tag = getenv("inven.view.content.tag");
$selector = $this->getSelector($response, $tag);
$selector = $this->getSelector($response, getenv("inven.view.content.tag"));
$media_urls = $this->getUrlsByMediaType($selector, "img", "src");
$media_urls = $this->getUrlsByMediaType($selector, "video", "src", $media_urls);
if ($this->getDebug()) {
if ($this->isDebug) {
throw new \Exception(sprintf(
"\n--------------%s Debug--------------\n%s%s\n---------------------------------------\n",
__FUNCTION__,
@ -68,6 +67,7 @@ class InvenCrawler extends MangboardCrawler
}
$this->backend_process($cnt, $listInfo, $storages);
}
log_message("notice", __FUNCTION__ . " 작업이 완료되었습니다.");
return $listInfo;
}
//리스트내용
@ -102,10 +102,9 @@ class InvenCrawler extends MangboardCrawler
public function execute(int $max_limit): void
{
try {
if ($this->getDebug()) {
if ($this->isDebug) {
$url = getenv("inven.view.test.url.{$this->_board_name}");
log_message("notice", __FUNCTION__ . "=> DEBUG 게시물 {$url} 작업시작");
$this->detail_page(1, ['detail_url' => $url]);
$this->detail_process(1, ['detail_url' => $url]);
log_message("notice", __FUNCTION__ . "=> DEBUG 게시물 {$url} 작업종료");
} else {
$listInfos = [];
@ -125,7 +124,7 @@ class InvenCrawler extends MangboardCrawler
if (!count($listInfos)) {
throw new \Exception("Target URL이 없습니다.");
}
$this->main_process($max_limit, $listInfos);
$this->list_process($max_limit, $listInfos);
}
log_message("notice", __FUNCTION__ . " 작업이 완료되었습니다.");
} catch (\Exception $e) {

29
app/Libraries/MyCrawler/Mangboard/MangboardCrawler.php
View File

@ -22,6 +22,7 @@ abstract class MangboardCrawler extends MyCrawler
$this->_board_name = $board_name;
$this->_user_entity = $user_entity;
}
abstract protected function detail_process(int $cnt, array $listInfo): array;
abstract public function execute(int $max_limit): void;
final protected function getMySocket()
{
@ -53,7 +54,7 @@ abstract class MangboardCrawler extends MyCrawler
);
foreach ($storages as $storage) {
try {
$storage->backend($boards_entity, $board_entity, $boardModel->getTable());
$storage->backend_process($boards_entity, $board_entity, $boardModel->getTable());
} catch (\Exception $e) {
log_message("notice", sprintf(
"\n---%s -> %s 게시물의 %s번째:%s 파일 등록 오류---\n%s\n--------------------------------\n",
@ -65,5 +66,31 @@ abstract class MangboardCrawler extends MyCrawler
));
}
}
log_message("notice", __FUNCTION__ . " 작업이 완료되었습니다.");
}
protected function list_process(int $max_limit, array $listInfos): void
{
//Limit가 0이면 $listInfos 갯수만큼 다하고, LIMIT 갯수 혹은 item의 갯수중 작은수만큼 한다.
$max_limit = !$max_limit || count($listInfos) <= $max_limit ? count($listInfos) : $max_limit;
$total = count($listInfos);
$i = 1;
foreach ($listInfos as $listInfo) {
if ($i <= $max_limit) {
log_message("notice", __FUNCTION__ . " 게시물 {$i}번째/총:{$total} {$listInfo["nickname"]} 작업시작");
try {
//listInfo는 title,작성자,작성시간등등의 정보를 가지고 있어 detail_process 처리 안에서 바뀔 수 있으므로 다시 반환 받는다.
$listInfo = $this->detail_process($i, $listInfo);
} catch (\Exception $e) {
log_message("warning", sprintf(
"\n---%s {$i}번째/총:{$total} 오류---\n%s\n-----------------------------------------\n",
__FUNCTION__,
$e->getMessage()
));
}
log_message("notice", __FUNCTION__ . " 게시물 {$i}번째/총:{$total} {$listInfo["nickname"]} 작업완료.");
$i++;
}
}
log_message("notice", __FUNCTION__ . " 작업이 완료되었습니다.");
}
}

17
app/Libraries/MyCrawler/Mangboard/SirCrawler.php
View File

@ -69,7 +69,7 @@ class SirCrawler extends MangboardCrawler
// </video>
// </div>
// <p>..</p><div style=
protected function detail_page(int $cnt, array $listInfo): array
protected function detail_process(int $cnt, array $listInfo): array
{
$response = $this->getMySocket()->getContent($listInfo['detail_url']);
//작성시간
@ -78,7 +78,7 @@ class SirCrawler extends MangboardCrawler
$listInfo['date'] = trim($selector->text());
$listInfo['date'] = DateTime::createFromFormat('Y.m.d H:i:s', $listInfo['date']);
$listInfo['date'] = $listInfo['date']->format('Y-m-d H:i:s');
// if ($this->getDebug()) {
// if ($this->isDebug) {
// throw new \Exception(
// sprintf(
// "\n--------------%s Debug--------------\n%s\n%s\n---------------------------------------\n",
@ -89,11 +89,10 @@ class SirCrawler extends MangboardCrawler
// );
// }
//작성내용
$tag = getenv("sir.view.content.tag");
$selector = $this->getSelector($response, $tag);
$selector = $this->getSelector($response, getenv("sir.view.content.tag"));
$media_urls = $this->getUrlsByMediaType($selector, "img", "src");
$media_urls = $this->getUrlsByMediaType($selector, "video", "src", $media_urls);
if ($this->getDebug()) {
if ($this->isDebug) {
throw new \Exception(sprintf(
"\n--------------%s Debug--------------\n%s%s\n---------------------------------------\n",
__FUNCTION__,
@ -107,6 +106,7 @@ class SirCrawler extends MangboardCrawler
}
$this->backend_process($cnt, $listInfo, $storages);
}
log_message("notice", __FUNCTION__ . " 작업이 완료되었습니다.");
return $listInfo;
}
//리스트내용
@ -142,10 +142,9 @@ class SirCrawler extends MangboardCrawler
public function execute(int $max_limit): void
{
try {
if ($this->getDebug()) {
if ($this->isDebug) {
$url = getenv("sir.view.test.url.{$this->_board_name}");
log_message("notice", __FUNCTION__ . "=> DEBUG 게시물 {$url} 작업시작");
$this->detail_page(1, ['detail_url' => $url]);
$this->detail_process(1, ['detail_url' => $url]);
log_message("notice", __FUNCTION__ . "=> DEBUG 게시물 {$url} 작업종료");
} else {
$listInfos = [];
@ -166,7 +165,7 @@ class SirCrawler extends MangboardCrawler
if (!count($listInfos)) {
throw new \Exception("Target URL이 없습니다.");
}
$this->main_process($max_limit, $listInfos);
$this->list_process($max_limit, $listInfos);
}
log_message("notice", __FUNCTION__ . " 작업이 완료되었습니다.");
} catch (\Exception $e) {

15
app/Libraries/MyCrawler/Mangboard/YamapCrawler.php
View File

@ -11,14 +11,13 @@ class YamapCrawler extends MangboardCrawler
{
parent::__construct($host, $board_name, $user_entity);
}
protected function detail_page(int $cnt, array $listInfo): array
protected function detail_process(int $cnt, array $listInfo): array
{
$response = $this->getMySocket()->getContent($listInfo['detail_url']);
$tag = getenv("yamap.view.content.tag");
$selector = $this->getSelector($response, $tag);
$selector = $this->getSelector($response, getenv("yamap.view.content.tag"));
$media_urls = $this->getUrlsByMediaType($selector, "img", "src");
$media_urls = $this->getUrlsByMediaType($selector, "video", "src", $media_urls);
if ($this->getDebug()) {
if ($this->isDebug) {
throw new \Exception(sprintf(
"\n--------------%s Debug--------------\n%s%s\n---------------------------------------\n",
__FUNCTION__,
@ -33,6 +32,7 @@ class YamapCrawler extends MangboardCrawler
}
$this->backend_process($cnt, $listInfo, $storages);
}
log_message("notice", __FUNCTION__ . " 작업이 완료되었습니다.");
return $listInfo;
}
//리스트내용
@ -68,10 +68,9 @@ class YamapCrawler extends MangboardCrawler
public function execute(int $max_limit): void
{
try {
if ($this->getDebug()) {
if ($this->isDebug) {
$url = getenv("yamap.view.test.url.{$this->_board_name}");
log_message("notice", __FUNCTION__ . "DEBUG 게시물 {$url} 작업시작");
$this->detail_page(1, ['detail_url' => $url]);
$this->detail_process(1, ['detail_url' => $url]);
log_message("notice", __FUNCTION__ . "DEBUG 게시물 {$url} 작업종료");
} else {
$listInfos = [];
@ -96,7 +95,7 @@ class YamapCrawler extends MangboardCrawler
if (!count($listInfos)) {
throw new \Exception("Target URL이 없습니다.");
}
$this->main_process($max_limit, $listInfos);
$this->list_process($max_limit, $listInfos);
}
log_message("notice", __FUNCTION__ . " 작업이 완료되었습니다.");
} catch (\Exception $e) {

19
app/Libraries/MyCrawler/Mangboard/YamoonCrawler.php
View File

@ -42,13 +42,13 @@ class YamoonCrawler extends MangboardCrawler
// </div>
// <div id="freesubframe"></div>
// </div>
protected function detail_page(int $cnt, array $listInfo): array
protected function detail_process(int $cnt, array $listInfo): array
{
$response = $this->getMySocket()->getContent("/newboard/yamoonboard/" . $listInfo['detail_url']);
//작성시간
// $selector = $this->getSelector($response, getenv("yamoon.view.date.tag"));
// $listInfo['date'] = trim($selector->text());
// if ($this->getDebug()) {
// if ($this->isDebug) {
// throw new \Exception(
// sprintf(
// "\n--------------%s Debug--------------\n%s\n%s\n---------------------------------------\n",
@ -59,11 +59,10 @@ class YamoonCrawler extends MangboardCrawler
// );
// }
//작성내용
$tag = getenv("yamoon.view.content.tag");
$selector = $this->getSelector($response, $tag);
$selector = $this->getSelector($response, getenv("yamoon.view.content.tag"));
$media_urls = $this->getUrlsByMediaType($selector, "img", "src");
$media_urls = $this->getUrlsByMediaType($selector, "video", "src", $media_urls);
if ($this->getDebug()) {
if ($this->isDebug) {
throw new \Exception(sprintf(
"\n--------------%s Debug--------------\n%s%s\n---------------------------------------\n",
__FUNCTION__,
@ -78,6 +77,7 @@ class YamoonCrawler extends MangboardCrawler
}
$this->backend_process($cnt, $listInfo, $storages);
}
log_message("notice", __FUNCTION__ . " 작업이 완료되었습니다.");
return $listInfo;
}
//리스트 내용
@ -90,11 +90,10 @@ class YamoonCrawler extends MangboardCrawler
public function execute(int $max_limit): void
{
try {
if ($this->getDebug()) {
if ($this->isDebug) {
$url = getenv("yamoon.view.test.url.{$this->_board_name}");
log_message("notice", __FUNCTION__ . "=> DEBUG 게시물 {$url} 작업시작");
$this->detail_page(1, ['detail_url' => $url]);
log_message("notice", __FUNCTION__ . "=> DEBUG 게시물 {$url} 작업종료");
$this->detail_process(1, ['detail_url' => $url]);
log_message("notice", __FUNCTION__ . "DEBUG 게시물 {$url} 작업종료");
} else {
$listInfos = [];
$response = $this->getMySocket()->getContent(getenv("yamoon.list.url.{$this->_board_name}"));
@ -111,7 +110,7 @@ class YamoonCrawler extends MangboardCrawler
if (!count($listInfos)) {
throw new \Exception("Target URL이 없습니다.");
}
$this->main_process($max_limit, $listInfos);
$this->list_process($max_limit, $listInfos);
}
log_message("notice", __FUNCTION__ . " 작업이 완료되었습니다.");
} catch (\Exception $e) {

45
app/Libraries/MyCrawler/MyCrawler.php
View File

@ -15,15 +15,14 @@ abstract class MyCrawler extends CommonLibrary
}
abstract protected function getMySocket();
abstract protected function createMyStorage();
abstract protected function detail_page(int $cnt, array $listInfo): array;
final protected function getSelector(string $content, string $tag): Crawler
{
$crawler = new Crawler($content);
if ($this->getDebug()) {
if ($this->isDebug) {
log_message("debug", __FUNCTION__ . "=> " . $tag);
}
$crawler->filter($tag);
if ($this->getDebug()) {
if ($this->isDebug) {
log_message("debug", sprintf(
"\n------------%s HTML-------------\n%s\n-----------------------------------------------------\n",
__FUNCTION__,
@ -107,17 +106,17 @@ abstract class MyCrawler extends CommonLibrary
}
final protected function media_process(array $media_urls): array
{
// log_message("debug", var_export($media_urls, true));
$file_sequence = 1;
$storages = []; //CreateBoard에서 사용을 위해 DetailPage마다 초기화
// log_message("debug", var_export($urls, true));
foreach ($media_urls as $media_type => $urls) {
$total = count($urls);
foreach ($urls as $url) {
log_message("notice", __FUNCTION__ . " {$file_sequence}번째/총:{$total} MediaType->{$media_type} 작업 시작");
try {
list($file_name, $content) = $this->media_download($media_type, $url);
$storages[] = $this->media_save($file_sequence, $media_type, $file_name, $content);
$storage = $this->media_save($file_sequence, $media_type, $file_name, $content);
log_message("debug", __FUNCTION__ . " {$file_sequence}번째/총:{$total} 결과=>" . $storage->getOriginName());
$storages[] = $storage;
} catch (\Exception $e) {
log_message("warning", sprintf(
"\n---%s MediaType->%s {$file_sequence}번째/총:{$total} 오류---\n%s\n-----------------------------------------\n",
@ -130,40 +129,6 @@ abstract class MyCrawler extends CommonLibrary
$file_sequence++;
}
}
$i = 1;
$total = count($storages);
foreach ($storages as $storage) {
log_message("debug", __FUNCTION__ . " {$i}번째/총:{$total} 결과=>" . $storage->getOriginName());
$i++;
}
return $storages;
}
protected function main_process(int $max_limit, array $listInfos): void
{
//Limit가 0이면 $listInfos 갯수만큼 다하고, LIMIT 갯수 혹은 item의 갯수중 작은수만큼 한다.
if ($max_limit) {
$max_limit = count($listInfos) <= $max_limit ? count($listInfos) : $max_limit;
} else {
$max_limit = count($listInfos);
}
$total = count($listInfos);
$i = 1;
foreach ($listInfos as $listInfo) {
if ($i <= $max_limit) {
log_message("notice", __FUNCTION__ . " 게시물 {$i}번째/총:{$total} {$listInfo["nickname"]} 작업시작");
try {
//listInfo는 title,작성자,작성시간등등의 정보를 가지고 있어 detail_page 처리 안에서 바뀔 수 있으므로 다시 반환 받는다.
$listInfo = $this->detail_page($i, $listInfo);
} catch (\Exception $e) {
log_message("warning", sprintf(
"\n---%s {$i}번째/총:{$total} 오류---\n%s\n-----------------------------------------\n",
__FUNCTION__,
$e->getMessage()
));
}
log_message("notice", __FUNCTION__ . " 게시물 {$i}번째/총:{$total} {$listInfo["nickname"]} 작업완료.");
$i++;
}
}
}
}

12
app/Libraries/MySocket/Web/GoogleSocket.php
View File

@ -19,12 +19,12 @@ class GoogleSocket extends WebSocket
public function getClient()
{
if (is_null($this->_client)) {
$this->_client = new \Google_Client();
$this->_client->setClientId(getenv("socket.google.client.id"));
$this->_client->setClientSecret(getenv("socket.google.client.key"));
$this->_client->setRedirectUri(base_url() . getenv("socket.google.client.callback_url"));
$this->_client->addScope('email');
$this->_client->addScope('profile');
// $this->_client = new \Google_Client();
// $this->_client->setClientId(getenv("socket.google.client.id"));
// $this->_client->setClientSecret(getenv("socket.google.client.key"));
// $this->_client->setRedirectUri(base_url() . getenv("socket.google.client.callback_url"));
// $this->_client->addScope('email');
// $this->_client->addScope('profile');
}
return $this->_client;
}

2
app/Libraries/MyStorage/MangboardStorage.php
View File

@ -74,7 +74,7 @@ class MangboardStorage extends FileStorage
return $content;
}
final public function backend(BoardsEntity $boards_entity, BoardEntity $board_entity, string $board_table)
final public function backend_process(BoardsEntity $boards_entity, BoardEntity $board_entity, string $board_table)
{
$this->getFileModel()->createByCrawler(
$boards_entity,

6
app/Models/Mangboard/BoardModel.php
View File

@ -154,9 +154,9 @@ class BoardModel extends CommonModel
UserEntity $user_entity,
int $cnt,
array $listInfo,
array $storages
array $storages = [],
array $formDatas = ['image_path' => "", 'content' => ""]
): BoardEntity {
$formDatas = [];
//미디어관련정보 entity에 넣기
$formDatas['title'] = $listInfo["title"];
$formDatas['user_pid'] = $user_entity->getPK();
@ -167,8 +167,6 @@ class BoardModel extends CommonModel
$formDatas['reg_date'] = date("Y-m-d H:i:s", strtotime($listInfo['date']));
$formDatas['data_type'] = "html";
$formDatas['editor_type'] = "S";
$formDatas['image_path'] = "";
$formDatas['content'] = "";
foreach ($storages as $storage) {
if ($formDatas['image_path'] == "") {
$formDatas['image_path'] = $storage->getBasePath() . DIRECTORY_SEPARATOR . $storage->getPath() . DIRECTORY_SEPARATOR . $storage->getOriginName();

5
app/Models/Mangboard/FileModel.php
View File

@ -130,10 +130,9 @@ class FileModel extends CommonModel
UserEntity $user_entity,
BoardEntity $board_entity,
string $board_table,
MangboardStorage $storage
MangboardStorage $storage,
array $formDatas = []
): FileEntity {
//파일관리 table에 등록
$formDatas = [];
//Board PID 넣기
$formDatas['board_pid'] = $board_entity->getPk();
$formDatas['user_pid'] = $user_entity->getPK();