idcjp/Automation
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Automation init...3

Browse Source
This commit is contained in:
최준흠 2024-09-18 17:19:48 +09:00
parent bceec3122e
commit 80b3e4bbc0
13 changed files with 92 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
app/Controllers/CommonController.php
View File

@ -6,18 +6,18 @@ use App\Controllers\BaseController;
abstract class CommonController extends BaseController abstract class CommonController extends BaseController
{ {
private $_datas = []; private $_options = [];
final public function __get($name): array|null final public function __get($name): array|null
{ {
if (!array_key_exists($name, $this->_datas)) { if (!array_key_exists($name, $this->_options)) {
return null; return null;
} }
return $this->_datas; return $this->_options;
} }
final public function __set($name, $value): void final public function __set($name, $value): void
{ {
$this->_datas[$name] = $value; $this->_options[$name] = $value;
} }
} }

8
app/Controllers/Mangboard/CrawlerController.php
View File

@ -54,7 +54,7 @@ class CrawlerController extends CommonController
$user_entity = $this->login(in_array('id', $params) ? $params['id'] : ""); $user_entity = $this->login(in_array('id', $params) ? $params['id'] : "");
//2. 필요한 로그인한 사용자정보,Socket,Storage 정의후 Crawler에게 전달. //2. 필요한 로그인한 사용자정보,Socket,Storage 정의후 Crawler에게 전달.
$crawler = new YamapCrawler(getenv('yamap.host.url'), $board_name, $user_entity); $crawler = new YamapCrawler(getenv('yamap.host.url'), $board_name, $user_entity);
$crawler->setDebug(in_array('debug', $params)); $crawler->isDebug = in_array('debug', $params);
$crawler->execute(intval(getenv("yamap.list.max_limit"))); $crawler->execute(intval(getenv("yamap.list.max_limit")));
return "완료되었습니다."; return "완료되었습니다.";
} catch (\Exception $e) { } catch (\Exception $e) {
@ -69,7 +69,7 @@ class CrawlerController extends CommonController
$user_entity = $this->login(in_array('id', $params) ? $params['id'] : ""); $user_entity = $this->login(in_array('id', $params) ? $params['id'] : "");
//2. 필요한 로그인한 사용자정보,Socket,Storage 정의후 Crawler에게 전달. //2. 필요한 로그인한 사용자정보,Socket,Storage 정의후 Crawler에게 전달.
$crawler = new YamoonCrawler(getenv("yamoon.host.url"), $board_name, $user_entity); $crawler = new YamoonCrawler(getenv("yamoon.host.url"), $board_name, $user_entity);
$crawler->setDebug(in_array('debug', $params)); $crawler->isDebug = in_array('debug', $params);
$crawler->execute(intval(getenv("yamoon.list.max_limit"))); $crawler->execute(intval(getenv("yamoon.list.max_limit")));
return "완료되었습니다."; return "완료되었습니다.";
} catch (\Exception $e) { } catch (\Exception $e) {
@ -84,7 +84,7 @@ class CrawlerController extends CommonController
$user_entity = $this->login(in_array('id', $params) ? $params['id'] : ""); $user_entity = $this->login(in_array('id', $params) ? $params['id'] : "");
//2. 필요한 로그인한 사용자정보,Socket,Storage 정의후 Crawler에게 전달. //2. 필요한 로그인한 사용자정보,Socket,Storage 정의후 Crawler에게 전달.
$crawler = new SirCrawler(getenv("sir.host.url"), $board_name, $user_entity); $crawler = new SirCrawler(getenv("sir.host.url"), $board_name, $user_entity);
$crawler->setDebug(in_array('debug', $params)); $crawler->isDebug = in_array('debug', $params);
$crawler->execute(intval(getenv("sir.list.max_limit"))); $crawler->execute(intval(getenv("sir.list.max_limit")));
return "완료되었습니다."; return "완료되었습니다.";
} catch (\Exception $e) { } catch (\Exception $e) {
@ -99,7 +99,7 @@ class CrawlerController extends CommonController
$user_entity = $this->login(in_array('id', $params) ? $params['id'] : ""); $user_entity = $this->login(in_array('id', $params) ? $params['id'] : "");
//2. 필요한 로그인한 사용자정보,Socket,Storage 정의후 Crawler에게 전달. //2. 필요한 로그인한 사용자정보,Socket,Storage 정의후 Crawler에게 전달.
$crawler = new InvenCrawler(getenv("inven.host.url"), $board_name, $user_entity); $crawler = new InvenCrawler(getenv("inven.host.url"), $board_name, $user_entity);
$crawler->setDebug(in_array('debug', $params)); $crawler->isDebug = in_array('debug', $params);
$crawler->execute(intval(getenv("inven.list.max_limit"))); $crawler->execute(intval(getenv("inven.list.max_limit")));
return "완료되었습니다."; return "완료되었습니다.";
} catch (\Exception $e) { } catch (\Exception $e) {

15
app/Libraries/CommonLibrary.php
View File

@ -4,15 +4,18 @@ namespace App\Libraries;
abstract class CommonLibrary abstract class CommonLibrary
{ {
private $_debug = false; private $_options = [];
protected function __construct() {} protected function __construct() {}
final public function __get($name): array|null
final public function getDebug(): bool
{ {
return $this->_debug; if (!array_key_exists($name, $this->_options)) {
return null;
}
return $this->_options;
} }
final public function setDebug(bool $debug): void
final public function __set($name, $value): void
{ {
$this->_debug = $debug; $this->_options[$name] = $value;
} }
} }

15
app/Libraries/MyCrawler/Mangboard/InvenCrawler.php
View File

@ -46,14 +46,13 @@ class InvenCrawler extends MangboardCrawler
// </div> // </div>
// <!-- ============== End CONTENT ============== --> // <!-- ============== End CONTENT ============== -->
// </div> // </div>
protected function detail_page(int $cnt, array $listInfo): array protected function detail_process(int $cnt, array $listInfo): array
{ {
$response = $this->getMySocket()->getContent($listInfo['detail_url']); $response = $this->getMySocket()->getContent($listInfo['detail_url']);
$tag = getenv("inven.view.content.tag"); $selector = $this->getSelector($response, getenv("inven.view.content.tag"));
$selector = $this->getSelector($response, $tag);
$media_urls = $this->getUrlsByMediaType($selector, "img", "src"); $media_urls = $this->getUrlsByMediaType($selector, "img", "src");
$media_urls = $this->getUrlsByMediaType($selector, "video", "src", $media_urls); $media_urls = $this->getUrlsByMediaType($selector, "video", "src", $media_urls);
if ($this->getDebug()) { if ($this->isDebug) {
throw new \Exception(sprintf( throw new \Exception(sprintf(
"\n--------------%s Debug--------------\n%s%s\n---------------------------------------\n", "\n--------------%s Debug--------------\n%s%s\n---------------------------------------\n",
__FUNCTION__, __FUNCTION__,
@ -68,6 +67,7 @@ class InvenCrawler extends MangboardCrawler
} }
$this->backend_process($cnt, $listInfo, $storages); $this->backend_process($cnt, $listInfo, $storages);
} }
log_message("notice", __FUNCTION__ . " 작업이 완료되었습니다.");
return $listInfo; return $listInfo;
} }
//리스트내용 //리스트내용
@ -102,10 +102,9 @@ class InvenCrawler extends MangboardCrawler
public function execute(int $max_limit): void public function execute(int $max_limit): void
{ {
try { try {
if ($this->getDebug()) { if ($this->isDebug) {
$url = getenv("inven.view.test.url.{$this->_board_name}"); $url = getenv("inven.view.test.url.{$this->_board_name}");
log_message("notice", __FUNCTION__ . "=> DEBUG 게시물 {$url} 작업시작"); $this->detail_process(1, ['detail_url' => $url]);
$this->detail_page(1, ['detail_url' => $url]);
log_message("notice", __FUNCTION__ . "=> DEBUG 게시물 {$url} 작업종료"); log_message("notice", __FUNCTION__ . "=> DEBUG 게시물 {$url} 작업종료");
} else { } else {
$listInfos = []; $listInfos = [];
@ -125,7 +124,7 @@ class InvenCrawler extends MangboardCrawler
if (!count($listInfos)) { if (!count($listInfos)) {
throw new \Exception("Target URL이 없습니다."); throw new \Exception("Target URL이 없습니다.");
} }
$this->main_process($max_limit, $listInfos); $this->list_process($max_limit, $listInfos);
} }
log_message("notice", __FUNCTION__ . " 작업이 완료되었습니다."); log_message("notice", __FUNCTION__ . " 작업이 완료되었습니다.");
} catch (\Exception $e) { } catch (\Exception $e) {

29
app/Libraries/MyCrawler/Mangboard/MangboardCrawler.php
View File

@ -22,6 +22,7 @@ abstract class MangboardCrawler extends MyCrawler
$this->_board_name = $board_name; $this->_board_name = $board_name;
$this->_user_entity = $user_entity; $this->_user_entity = $user_entity;
} }
abstract protected function detail_process(int $cnt, array $listInfo): array;
abstract public function execute(int $max_limit): void; abstract public function execute(int $max_limit): void;
final protected function getMySocket() final protected function getMySocket()
{ {
@ -53,7 +54,7 @@ abstract class MangboardCrawler extends MyCrawler
); );
foreach ($storages as $storage) { foreach ($storages as $storage) {
try { try {
$storage->backend($boards_entity, $board_entity, $boardModel->getTable()); $storage->backend_process($boards_entity, $board_entity, $boardModel->getTable());
} catch (\Exception $e) { } catch (\Exception $e) {
log_message("notice", sprintf( log_message("notice", sprintf(
"\n---%s -> %s 게시물의 %s번째:%s 파일 등록 오류---\n%s\n--------------------------------\n", "\n---%s -> %s 게시물의 %s번째:%s 파일 등록 오류---\n%s\n--------------------------------\n",
@ -65,5 +66,31 @@ abstract class MangboardCrawler extends MyCrawler
)); ));
} }
} }
log_message("notice", __FUNCTION__ . " 작업이 완료되었습니다.");
}
protected function list_process(int $max_limit, array $listInfos): void
{
//Limit가 0이면 $listInfos 갯수만큼 다하고, LIMIT 갯수 혹은 item의 갯수중 작은수만큼 한다.
$max_limit = !$max_limit || count($listInfos) <= $max_limit ? count($listInfos) : $max_limit;
$total = count($listInfos);
$i = 1;
foreach ($listInfos as $listInfo) {
if ($i <= $max_limit) {
log_message("notice", __FUNCTION__ . " 게시물 {$i}번째/총:{$total} {$listInfo["nickname"]} 작업시작");
try {
//listInfo는 title,작성자,작성시간등등의 정보를 가지고 있어 detail_process 처리 안에서 바뀔 수 있으므로 다시 반환 받는다.
$listInfo = $this->detail_process($i, $listInfo);
} catch (\Exception $e) {
log_message("warning", sprintf(
"\n---%s {$i}번째/총:{$total} 오류---\n%s\n-----------------------------------------\n",
__FUNCTION__,
$e->getMessage()
));
}
log_message("notice", __FUNCTION__ . " 게시물 {$i}번째/총:{$total} {$listInfo["nickname"]} 작업완료.");
$i++;
}
}
log_message("notice", __FUNCTION__ . " 작업이 완료되었습니다.");
} }
} }

17
app/Libraries/MyCrawler/Mangboard/SirCrawler.php
View File

@ -69,7 +69,7 @@ class SirCrawler extends MangboardCrawler
// </video> // </video>
// </div> // </div>
// <p>..</p><div style= // <p>..</p><div style=
protected function detail_page(int $cnt, array $listInfo): array protected function detail_process(int $cnt, array $listInfo): array
{ {
$response = $this->getMySocket()->getContent($listInfo['detail_url']); $response = $this->getMySocket()->getContent($listInfo['detail_url']);
//작성시간 //작성시간
@ -78,7 +78,7 @@ class SirCrawler extends MangboardCrawler
$listInfo['date'] = trim($selector->text()); $listInfo['date'] = trim($selector->text());
$listInfo['date'] = DateTime::createFromFormat('Y.m.d H:i:s', $listInfo['date']); $listInfo['date'] = DateTime::createFromFormat('Y.m.d H:i:s', $listInfo['date']);
$listInfo['date'] = $listInfo['date']->format('Y-m-d H:i:s'); $listInfo['date'] = $listInfo['date']->format('Y-m-d H:i:s');
// if ($this->getDebug()) { // if ($this->isDebug) {
// throw new \Exception( // throw new \Exception(
// sprintf( // sprintf(
// "\n--------------%s Debug--------------\n%s\n%s\n---------------------------------------\n", // "\n--------------%s Debug--------------\n%s\n%s\n---------------------------------------\n",
@ -89,11 +89,10 @@ class SirCrawler extends MangboardCrawler
// ); // );
// } // }
//작성내용 //작성내용
$tag = getenv("sir.view.content.tag"); $selector = $this->getSelector($response, getenv("sir.view.content.tag"));
$selector = $this->getSelector($response, $tag);
$media_urls = $this->getUrlsByMediaType($selector, "img", "src"); $media_urls = $this->getUrlsByMediaType($selector, "img", "src");
$media_urls = $this->getUrlsByMediaType($selector, "video", "src", $media_urls); $media_urls = $this->getUrlsByMediaType($selector, "video", "src", $media_urls);
if ($this->getDebug()) { if ($this->isDebug) {
throw new \Exception(sprintf( throw new \Exception(sprintf(
"\n--------------%s Debug--------------\n%s%s\n---------------------------------------\n", "\n--------------%s Debug--------------\n%s%s\n---------------------------------------\n",
__FUNCTION__, __FUNCTION__,
@ -107,6 +106,7 @@ class SirCrawler extends MangboardCrawler
} }
$this->backend_process($cnt, $listInfo, $storages); $this->backend_process($cnt, $listInfo, $storages);
} }
log_message("notice", __FUNCTION__ . " 작업이 완료되었습니다.");
return $listInfo; return $listInfo;
} }
//리스트내용 //리스트내용
@ -142,10 +142,9 @@ class SirCrawler extends MangboardCrawler
public function execute(int $max_limit): void public function execute(int $max_limit): void
{ {
try { try {
if ($this->getDebug()) { if ($this->isDebug) {
$url = getenv("sir.view.test.url.{$this->_board_name}"); $url = getenv("sir.view.test.url.{$this->_board_name}");
log_message("notice", __FUNCTION__ . "=> DEBUG 게시물 {$url} 작업시작"); $this->detail_process(1, ['detail_url' => $url]);
$this->detail_page(1, ['detail_url' => $url]);
log_message("notice", __FUNCTION__ . "=> DEBUG 게시물 {$url} 작업종료"); log_message("notice", __FUNCTION__ . "=> DEBUG 게시물 {$url} 작업종료");
} else { } else {
$listInfos = []; $listInfos = [];
@ -166,7 +165,7 @@ class SirCrawler extends MangboardCrawler
if (!count($listInfos)) { if (!count($listInfos)) {
throw new \Exception("Target URL이 없습니다."); throw new \Exception("Target URL이 없습니다.");
} }
$this->main_process($max_limit, $listInfos); $this->list_process($max_limit, $listInfos);
} }
log_message("notice", __FUNCTION__ . " 작업이 완료되었습니다."); log_message("notice", __FUNCTION__ . " 작업이 완료되었습니다.");
} catch (\Exception $e) { } catch (\Exception $e) {

15
app/Libraries/MyCrawler/Mangboard/YamapCrawler.php
View File

@ -11,14 +11,13 @@ class YamapCrawler extends MangboardCrawler
{ {
parent::__construct($host, $board_name, $user_entity); parent::__construct($host, $board_name, $user_entity);
} }
protected function detail_page(int $cnt, array $listInfo): array protected function detail_process(int $cnt, array $listInfo): array
{ {
$response = $this->getMySocket()->getContent($listInfo['detail_url']); $response = $this->getMySocket()->getContent($listInfo['detail_url']);
$tag = getenv("yamap.view.content.tag"); $selector = $this->getSelector($response, getenv("yamap.view.content.tag"));
$selector = $this->getSelector($response, $tag);
$media_urls = $this->getUrlsByMediaType($selector, "img", "src"); $media_urls = $this->getUrlsByMediaType($selector, "img", "src");
$media_urls = $this->getUrlsByMediaType($selector, "video", "src", $media_urls); $media_urls = $this->getUrlsByMediaType($selector, "video", "src", $media_urls);
if ($this->getDebug()) { if ($this->isDebug) {
throw new \Exception(sprintf( throw new \Exception(sprintf(
"\n--------------%s Debug--------------\n%s%s\n---------------------------------------\n", "\n--------------%s Debug--------------\n%s%s\n---------------------------------------\n",
__FUNCTION__, __FUNCTION__,
@ -33,6 +32,7 @@ class YamapCrawler extends MangboardCrawler
} }
$this->backend_process($cnt, $listInfo, $storages); $this->backend_process($cnt, $listInfo, $storages);
} }
log_message("notice", __FUNCTION__ . " 작업이 완료되었습니다.");
return $listInfo; return $listInfo;
} }
//리스트내용 //리스트내용
@ -68,10 +68,9 @@ class YamapCrawler extends MangboardCrawler
public function execute(int $max_limit): void public function execute(int $max_limit): void
{ {
try { try {
if ($this->getDebug()) { if ($this->isDebug) {
$url = getenv("yamap.view.test.url.{$this->_board_name}"); $url = getenv("yamap.view.test.url.{$this->_board_name}");
log_message("notice", __FUNCTION__ . "DEBUG 게시물 {$url} 작업시작"); $this->detail_process(1, ['detail_url' => $url]);
$this->detail_page(1, ['detail_url' => $url]);
log_message("notice", __FUNCTION__ . "DEBUG 게시물 {$url} 작업종료"); log_message("notice", __FUNCTION__ . "DEBUG 게시물 {$url} 작업종료");
} else { } else {
$listInfos = []; $listInfos = [];
@ -96,7 +95,7 @@ class YamapCrawler extends MangboardCrawler
if (!count($listInfos)) { if (!count($listInfos)) {
throw new \Exception("Target URL이 없습니다."); throw new \Exception("Target URL이 없습니다.");
} }
$this->main_process($max_limit, $listInfos); $this->list_process($max_limit, $listInfos);
} }
log_message("notice", __FUNCTION__ . " 작업이 완료되었습니다."); log_message("notice", __FUNCTION__ . " 작업이 완료되었습니다.");
} catch (\Exception $e) { } catch (\Exception $e) {

19
app/Libraries/MyCrawler/Mangboard/YamoonCrawler.php
View File

@ -42,13 +42,13 @@ class YamoonCrawler extends MangboardCrawler
// </div> // </div>
// <div id="freesubframe"></div> // <div id="freesubframe"></div>
// </div> // </div>
protected function detail_page(int $cnt, array $listInfo): array protected function detail_process(int $cnt, array $listInfo): array
{ {
$response = $this->getMySocket()->getContent("/newboard/yamoonboard/" . $listInfo['detail_url']); $response = $this->getMySocket()->getContent("/newboard/yamoonboard/" . $listInfo['detail_url']);
//작성시간 //작성시간
// $selector = $this->getSelector($response, getenv("yamoon.view.date.tag")); // $selector = $this->getSelector($response, getenv("yamoon.view.date.tag"));
// $listInfo['date'] = trim($selector->text()); // $listInfo['date'] = trim($selector->text());
// if ($this->getDebug()) { // if ($this->isDebug) {
// throw new \Exception( // throw new \Exception(
// sprintf( // sprintf(
// "\n--------------%s Debug--------------\n%s\n%s\n---------------------------------------\n", // "\n--------------%s Debug--------------\n%s\n%s\n---------------------------------------\n",
@ -59,11 +59,10 @@ class YamoonCrawler extends MangboardCrawler
// ); // );
// } // }
//작성내용 //작성내용
$tag = getenv("yamoon.view.content.tag"); $selector = $this->getSelector($response, getenv("yamoon.view.content.tag"));
$selector = $this->getSelector($response, $tag);
$media_urls = $this->getUrlsByMediaType($selector, "img", "src"); $media_urls = $this->getUrlsByMediaType($selector, "img", "src");
$media_urls = $this->getUrlsByMediaType($selector, "video", "src", $media_urls); $media_urls = $this->getUrlsByMediaType($selector, "video", "src", $media_urls);
if ($this->getDebug()) { if ($this->isDebug) {
throw new \Exception(sprintf( throw new \Exception(sprintf(
"\n--------------%s Debug--------------\n%s%s\n---------------------------------------\n", "\n--------------%s Debug--------------\n%s%s\n---------------------------------------\n",
__FUNCTION__, __FUNCTION__,
@ -78,6 +77,7 @@ class YamoonCrawler extends MangboardCrawler
} }
$this->backend_process($cnt, $listInfo, $storages); $this->backend_process($cnt, $listInfo, $storages);
} }
log_message("notice", __FUNCTION__ . " 작업이 완료되었습니다.");
return $listInfo; return $listInfo;
} }
//리스트 내용 //리스트 내용
@ -90,11 +90,10 @@ class YamoonCrawler extends MangboardCrawler
public function execute(int $max_limit): void public function execute(int $max_limit): void
{ {
try { try {
if ($this->getDebug()) { if ($this->isDebug) {
$url = getenv("yamoon.view.test.url.{$this->_board_name}"); $url = getenv("yamoon.view.test.url.{$this->_board_name}");
log_message("notice", __FUNCTION__ . "=> DEBUG 게시물 {$url} 작업시작"); $this->detail_process(1, ['detail_url' => $url]);
$this->detail_page(1, ['detail_url' => $url]); log_message("notice", __FUNCTION__ . "DEBUG 게시물 {$url} 작업종료");
log_message("notice", __FUNCTION__ . "=> DEBUG 게시물 {$url} 작업종료");
} else { } else {
$listInfos = []; $listInfos = [];
$response = $this->getMySocket()->getContent(getenv("yamoon.list.url.{$this->_board_name}")); $response = $this->getMySocket()->getContent(getenv("yamoon.list.url.{$this->_board_name}"));
@ -111,7 +110,7 @@ class YamoonCrawler extends MangboardCrawler
if (!count($listInfos)) { if (!count($listInfos)) {
throw new \Exception("Target URL이 없습니다."); throw new \Exception("Target URL이 없습니다.");
} }
$this->main_process($max_limit, $listInfos); $this->list_process($max_limit, $listInfos);
} }
log_message("notice", __FUNCTION__ . " 작업이 완료되었습니다."); log_message("notice", __FUNCTION__ . " 작업이 완료되었습니다.");
} catch (\Exception $e) { } catch (\Exception $e) {

45
app/Libraries/MyCrawler/MyCrawler.php
View File

@ -15,15 +15,14 @@ abstract class MyCrawler extends CommonLibrary
} }
abstract protected function getMySocket(); abstract protected function getMySocket();
abstract protected function createMyStorage(); abstract protected function createMyStorage();
abstract protected function detail_page(int $cnt, array $listInfo): array;
final protected function getSelector(string $content, string $tag): Crawler final protected function getSelector(string $content, string $tag): Crawler
{ {
$crawler = new Crawler($content); $crawler = new Crawler($content);
if ($this->getDebug()) { if ($this->isDebug) {
log_message("debug", __FUNCTION__ . "=> " . $tag); log_message("debug", __FUNCTION__ . "=> " . $tag);
} }
$crawler->filter($tag); $crawler->filter($tag);
if ($this->getDebug()) { if ($this->isDebug) {
log_message("debug", sprintf( log_message("debug", sprintf(
"\n------------%s HTML-------------\n%s\n-----------------------------------------------------\n", "\n------------%s HTML-------------\n%s\n-----------------------------------------------------\n",
__FUNCTION__, __FUNCTION__,
@ -107,17 +106,17 @@ abstract class MyCrawler extends CommonLibrary
} }
final protected function media_process(array $media_urls): array final protected function media_process(array $media_urls): array
{ {
// log_message("debug", var_export($media_urls, true));
$file_sequence = 1; $file_sequence = 1;
$storages = []; //CreateBoard에서 사용을 위해 DetailPage마다 초기화 $storages = []; //CreateBoard에서 사용을 위해 DetailPage마다 초기화
// log_message("debug", var_export($urls, true));
foreach ($media_urls as $media_type => $urls) { foreach ($media_urls as $media_type => $urls) {
$total = count($urls); $total = count($urls);
foreach ($urls as $url) { foreach ($urls as $url) {
log_message("notice", __FUNCTION__ . " {$file_sequence}번째/총:{$total} MediaType->{$media_type} 작업 시작"); log_message("notice", __FUNCTION__ . " {$file_sequence}번째/총:{$total} MediaType->{$media_type} 작업 시작");
try { try {
list($file_name, $content) = $this->media_download($media_type, $url); list($file_name, $content) = $this->media_download($media_type, $url);
$storages[] = $this->media_save($file_sequence, $media_type, $file_name, $content); $storage = $this->media_save($file_sequence, $media_type, $file_name, $content);
log_message("debug", __FUNCTION__ . " {$file_sequence}번째/총:{$total} 결과=>" . $storage->getOriginName());
$storages[] = $storage;
} catch (\Exception $e) { } catch (\Exception $e) {
log_message("warning", sprintf( log_message("warning", sprintf(
"\n---%s MediaType->%s {$file_sequence}번째/총:{$total} 오류---\n%s\n-----------------------------------------\n", "\n---%s MediaType->%s {$file_sequence}번째/총:{$total} 오류---\n%s\n-----------------------------------------\n",
@ -130,40 +129,6 @@ abstract class MyCrawler extends CommonLibrary
$file_sequence++; $file_sequence++;
} }
} }
$i = 1;
$total = count($storages);
foreach ($storages as $storage) {
log_message("debug", __FUNCTION__ . " {$i}번째/총:{$total} 결과=>" . $storage->getOriginName());
$i++;
}
return $storages; return $storages;
} }
protected function main_process(int $max_limit, array $listInfos): void
{
//Limit가 0이면 $listInfos 갯수만큼 다하고, LIMIT 갯수 혹은 item의 갯수중 작은수만큼 한다.
if ($max_limit) {
$max_limit = count($listInfos) <= $max_limit ? count($listInfos) : $max_limit;
} else {
$max_limit = count($listInfos);
}
$total = count($listInfos);
$i = 1;
foreach ($listInfos as $listInfo) {
if ($i <= $max_limit) {
log_message("notice", __FUNCTION__ . " 게시물 {$i}번째/총:{$total} {$listInfo["nickname"]} 작업시작");
try {
//listInfo는 title,작성자,작성시간등등의 정보를 가지고 있어 detail_page 처리 안에서 바뀔 수 있으므로 다시 반환 받는다.
$listInfo = $this->detail_page($i, $listInfo);
} catch (\Exception $e) {
log_message("warning", sprintf(
"\n---%s {$i}번째/총:{$total} 오류---\n%s\n-----------------------------------------\n",
__FUNCTION__,
$e->getMessage()
));
}
log_message("notice", __FUNCTION__ . " 게시물 {$i}번째/총:{$total} {$listInfo["nickname"]} 작업완료.");
$i++;
}
}
}
} }

12
app/Libraries/MySocket/Web/GoogleSocket.php
View File

@ -19,12 +19,12 @@ class GoogleSocket extends WebSocket
public function getClient() public function getClient()
{ {
if (is_null($this->_client)) { if (is_null($this->_client)) {
$this->_client = new \Google_Client(); // $this->_client = new \Google_Client();
$this->_client->setClientId(getenv("socket.google.client.id")); // $this->_client->setClientId(getenv("socket.google.client.id"));
$this->_client->setClientSecret(getenv("socket.google.client.key")); // $this->_client->setClientSecret(getenv("socket.google.client.key"));
$this->_client->setRedirectUri(base_url() . getenv("socket.google.client.callback_url")); // $this->_client->setRedirectUri(base_url() . getenv("socket.google.client.callback_url"));
$this->_client->addScope('email'); // $this->_client->addScope('email');
$this->_client->addScope('profile'); // $this->_client->addScope('profile');
} }
return $this->_client; return $this->_client;
} }

2
app/Libraries/MyStorage/MangboardStorage.php
View File

@ -74,7 +74,7 @@ class MangboardStorage extends FileStorage
return $content; return $content;
} }
final public function backend(BoardsEntity $boards_entity, BoardEntity $board_entity, string $board_table) final public function backend_process(BoardsEntity $boards_entity, BoardEntity $board_entity, string $board_table)
{ {
$this->getFileModel()->createByCrawler( $this->getFileModel()->createByCrawler(
$boards_entity, $boards_entity,

6
app/Models/Mangboard/BoardModel.php
View File

@ -154,9 +154,9 @@ class BoardModel extends CommonModel
UserEntity $user_entity, UserEntity $user_entity,
int $cnt, int $cnt,
array $listInfo, array $listInfo,
array $storages array $storages = [],
array $formDatas = ['image_path' => "", 'content' => ""]
): BoardEntity { ): BoardEntity {
$formDatas = [];
//미디어관련정보 entity에 넣기 //미디어관련정보 entity에 넣기
$formDatas['title'] = $listInfo["title"]; $formDatas['title'] = $listInfo["title"];
$formDatas['user_pid'] = $user_entity->getPK(); $formDatas['user_pid'] = $user_entity->getPK();
@ -167,8 +167,6 @@ class BoardModel extends CommonModel
$formDatas['reg_date'] = date("Y-m-d H:i:s", strtotime($listInfo['date'])); $formDatas['reg_date'] = date("Y-m-d H:i:s", strtotime($listInfo['date']));
$formDatas['data_type'] = "html"; $formDatas['data_type'] = "html";
$formDatas['editor_type'] = "S"; $formDatas['editor_type'] = "S";
$formDatas['image_path'] = "";
$formDatas['content'] = "";
foreach ($storages as $storage) { foreach ($storages as $storage) {
if ($formDatas['image_path'] == "") { if ($formDatas['image_path'] == "") {
$formDatas['image_path'] = $storage->getBasePath() . DIRECTORY_SEPARATOR . $storage->getPath() . DIRECTORY_SEPARATOR . $storage->getOriginName(); $formDatas['image_path'] = $storage->getBasePath() . DIRECTORY_SEPARATOR . $storage->getPath() . DIRECTORY_SEPARATOR . $storage->getOriginName();

5
app/Models/Mangboard/FileModel.php
View File

@ -130,10 +130,9 @@ class FileModel extends CommonModel
UserEntity $user_entity, UserEntity $user_entity,
BoardEntity $board_entity, BoardEntity $board_entity,
string $board_table, string $board_table,
MangboardStorage $storage MangboardStorage $storage,
array $formDatas = []
): FileEntity { ): FileEntity {
//파일관리 table에 등록
$formDatas = [];
//Board PID 넣기 //Board PID 넣기
$formDatas['board_pid'] = $board_entity->getPk(); $formDatas['board_pid'] = $board_entity->getPk();
$formDatas['user_pid'] = $user_entity->getPK(); $formDatas['user_pid'] = $user_entity->getPK();