61 lines
1.9 KiB
PHP
61 lines
1.9 KiB
PHP
<?php
|
|
|
|
namespace App\Libraries\MyCrawler;
|
|
|
|
use App\Libraries\CommonLibrary;
|
|
use Symfony\Component\DomCrawler\Crawler;
|
|
|
|
abstract class MyCrawlerLibrary extends CommonLibrary
|
|
{
|
|
protected function __construct()
|
|
{
|
|
parent::__construct();
|
|
}
|
|
abstract public function getMySocket();
|
|
abstract public function getMyStorage();
|
|
|
|
final protected function getContent(string $url, string $tag): Crawler
|
|
{
|
|
$response = $this->getMySocket()->getContent($url);
|
|
if (!$response) {
|
|
throw new \Exception("getCrawler 실패:{$url}");
|
|
}
|
|
$crawler = new Crawler($response);
|
|
return $crawler->filter($tag);
|
|
}
|
|
|
|
private function getNodes(Crawler $crawler, array $options, $nodes = []): array
|
|
{
|
|
$crawler->filter($options["tag"])->each(
|
|
function (Crawler $node) use (&$options, &$nodes): void {
|
|
log_message("debug", sprintf("getNode->%s[%s]", $options["tag"], $node->attr($options['attr'])));
|
|
$nodes[] = $node;
|
|
}
|
|
);
|
|
return $nodes;
|
|
}
|
|
|
|
|
|
//Download한 파일 저장후 추가작업시 사용
|
|
protected function download_process($entity, int $file_sequence)
|
|
{
|
|
return $entity;
|
|
}
|
|
|
|
final protected function download(string $mediaType, Crawler $crawler, array $options, array $entitys = []): array
|
|
{
|
|
$nodes = $this->getNodes($crawler, $options);
|
|
$file_sequence = 1;
|
|
foreach ($nodes as $node) {
|
|
list($fileName, $content) = $this->getMySocket()->download($node->attr($options["attr"]));
|
|
$entity = $this->getMyStorage()->save($fileName, $mediaType, $content, $file_sequence);
|
|
if ($entity === null) {
|
|
continue;
|
|
}
|
|
$entitys[] = $this->download_process($entity, $file_sequence);
|
|
$file_sequence++;
|
|
}
|
|
return $entitys;
|
|
}
|
|
}
|