_doc = new DOMDocument(); $this->_doc->substituteEntities = true; if ($isFile) { $htmlData = file_get_contents($data); } else { $htmlData = $data; } @$this->_doc->loadHTML($htmlData); if ($this->_doc->encoding === null) { // Document encoding is not recognized /** @todo improve HTML vs HTML fragment recognition */ if (preg_match('//i', $htmlData, $matches, PREG_OFFSET_CAPTURE)) { // It's an HTML document // Add additional HEAD section and recognize document $htmlTagOffset = $matches[0][1] + strlen($matches[0][0]); @$this->_doc->loadHTML(iconv($defaultEncoding, 'UTF-8//IGNORE', substr($htmlData, 0, $htmlTagOffset)) . '' . iconv($defaultEncoding, 'UTF-8//IGNORE', substr($htmlData, $htmlTagOffset))); // Remove additional HEAD section $xpath = new DOMXPath($this->_doc); $head = $xpath->query('/html/head')->item(0); $head->parentNode->removeChild($head); } else { // It's an HTML fragment @$this->_doc->loadHTML('' . iconv($defaultEncoding, 'UTF-8//IGNORE', $htmlData) . ''); } } /** @todo Add correction of wrong HTML encoding recognition processing * The case is: * Content-type HTTP-EQUIV meta tag is presented, but ISO-8859-5 encoding is actually used, * even $this->_doc->encoding demonstrates another recognized encoding */ $xpath = new DOMXPath($this->_doc); $docTitle = ''; $titleNodes = $xpath->query('/html/head/title'); foreach ($titleNodes as $titleNode) { // title should always have only one entry, but we process all nodeset entries $docTitle .= $titleNode->nodeValue . ' '; } $this->addField(Zend_Search_Lucene_Field::Text('title', $docTitle, 'UTF-8')); $metaNodes = $xpath->query('/html/head/meta[@name]'); foreach ($metaNodes as $metaNode) { $this->addField(Zend_Search_Lucene_Field::Text($metaNode->getAttribute('name'), $metaNode->getAttribute('content'), 'UTF-8')); } $docBody = ''; $bodyNodes = $xpath->query('/html/body'); foreach ($bodyNodes as $bodyNode) { // body should always have only one entry, but we process all nodeset entries $this->_retrieveNodeText($bodyNode, $docBody); } if ($storeContent) { $this->addField(Zend_Search_Lucene_Field::Text('body', $docBody, 'UTF-8')); } else { $this->addField(Zend_Search_Lucene_Field::UnStored('body', $docBody, 'UTF-8')); } $linkNodes = $this->_doc->getElementsByTagName('a'); foreach ($linkNodes as $linkNode) { if (($href = $linkNode->getAttribute('href')) != '' && (!self::$_excludeNoFollowLinks || strtolower($linkNode->getAttribute('rel')) != 'nofollow' ) ) { $this->_links[] = $href; } } $this->_links = array_unique($this->_links); $linkNodes = $xpath->query('/html/head/link'); foreach ($linkNodes as $linkNode) { if (($href = $linkNode->getAttribute('href')) != '') { $this->_headerLinks[] = $href; } } $this->_headerLinks = array_unique($this->_headerLinks); } /** * Set exclude nofollow links flag * * @param boolean $newValue */ public static function setExcludeNoFollowLinks($newValue) { self::$_excludeNoFollowLinks = $newValue; } /** * Get exclude nofollow links flag * * @return boolean */ public static function getExcludeNoFollowLinks() { return self::$_excludeNoFollowLinks; } /** * Get node text * * We should exclude scripts, which may be not included into comment tags, CDATA sections, * * @param DOMNode $node * @param string &$text */ private function _retrieveNodeText(DOMNode $node, &$text) { if ($node->nodeType == XML_TEXT_NODE) { $text .= $node->nodeValue ; $text .= ' '; } else if ($node->nodeType == XML_ELEMENT_NODE && $node->nodeName != 'script') { foreach ($node->childNodes as $childNode) { $this->_retrieveNodeText($childNode, $text); } } } /** * Get document HREF links * * @return array */ public function getLinks() { return $this->_links; } /** * Get document header links * * @return array */ public function getHeaderLinks() { return $this->_headerLinks; } /** * Load HTML document from a string * * @param string $data * @param boolean $storeContent * @param string $defaultEncoding HTML encoding, is used if it's not specified using Content-type HTTP-EQUIV meta tag. * @return Zend_Search_Lucene_Document_Html */ public static function loadHTML($data, $storeContent = false, $defaultEncoding = '') { return new Zend_Search_Lucene_Document_Html($data, false, $storeContent, $defaultEncoding); } /** * Load HTML document from a file * * @param string $file * @param boolean $storeContent * @param string $defaultEncoding HTML encoding, is used if it's not specified using Content-type HTTP-EQUIV meta tag. * @return Zend_Search_Lucene_Document_Html */ public static function loadHTMLFile($file, $storeContent = false, $defaultEncoding = '') { return new Zend_Search_Lucene_Document_Html($file, true, $storeContent, $defaultEncoding); } /** * Highlight text in text node * * @param DOMText $node * @param array $wordsToHighlight * @param callback $callback Callback method, used to transform (highlighting) text. * @param array $params Array of additionall callback parameters (first non-optional parameter is a text to transform) * @throws Zend_Search_Lucene_Exception */ protected function _highlightTextNode(DOMText $node, $wordsToHighlight, $callback, $params) { $analyzer = Zend_Search_Lucene_Analysis_Analyzer::getDefault(); $analyzer->setInput($node->nodeValue, 'UTF-8'); $matchedTokens = array(); while (($token = $analyzer->nextToken()) !== null) { if (isset($wordsToHighlight[$token->getTermText()])) { $matchedTokens[] = $token; } } if (count($matchedTokens) == 0) { return; } $matchedTokens = array_reverse($matchedTokens); foreach ($matchedTokens as $token) { // Cut text after matched token $node->splitText($token->getEndOffset()); // Cut matched node $matchedWordNode = $node->splitText($token->getStartOffset()); // Retrieve HTML string representation for highlihted word $fullCallbackparamsList = $params; array_unshift($fullCallbackparamsList, $matchedWordNode->nodeValue); $highlightedWordNodeSetHtml = call_user_func_array($callback, $fullCallbackparamsList); // Transform HTML string to a DOM representation and automatically transform retrieved string // into valid XHTML (It's automatically done by loadHTML() method) $highlightedWordNodeSetDomDocument = new DOMDocument('1.0', 'UTF-8'); $success = @$highlightedWordNodeSetDomDocument-> loadHTML('' . $highlightedWordNodeSetHtml . ''); if (!$success) { require_once 'Zend/Search/Lucene/Exception.php'; throw new Zend_Search_Lucene_Exception("Error occured while loading highlighted text fragment: '$highlightedNodeHtml'."); } $highlightedWordNodeSetXpath = new DOMXPath($highlightedWordNodeSetDomDocument); $highlightedWordNodeSet = $highlightedWordNodeSetXpath->query('/html/body')->item(0)->childNodes; for ($count = 0; $count < $highlightedWordNodeSet->length; $count++) { $nodeToImport = $highlightedWordNodeSet->item($count); $node->parentNode->insertBefore($this->_doc->importNode($nodeToImport, true /* deep copy */), $matchedWordNode); } $node->parentNode->removeChild($matchedWordNode); } } /** * highlight words in content of the specified node * * @param DOMNode $contextNode * @param array $wordsToHighlight * @param callback $callback Callback method, used to transform (highlighting) text. * @param array $params Array of additionall callback parameters (first non-optional parameter is a text to transform) */ protected function _highlightNodeRecursive(DOMNode $contextNode, $wordsToHighlight, $callback, $params) { $textNodes = array(); if (!$contextNode->hasChildNodes()) { return; } foreach ($contextNode->childNodes as $childNode) { if ($childNode->nodeType == XML_TEXT_NODE) { // process node later to leave childNodes structure untouched $textNodes[] = $childNode; } else { // Process node if it's not a script node if ($childNode->nodeName != 'script') { $this->_highlightNodeRecursive($childNode, $wordsToHighlight, $callback, $params); } } } foreach ($textNodes as $textNode) { $this->_highlightTextNode($textNode, $wordsToHighlight, $callback, $params); } } /** * Standard callback method used to highlight words. * * @param string $stringToHighlight * @return string * @internal */ public function applyColour($stringToHighlight, $colour) { return '' . $stringToHighlight . ''; } /** * Highlight text with specified color * * @param string|array $words * @param string $colour * @return string */ public function highlight($words, $colour = '#66ffff') { return $this->highlightExtended($words, array($this, 'applyColour'), array($colour)); } /** * Highlight text using specified View helper or callback function. * * @param string|array $words Words to highlight. Words could be organized using the array or string. * @param callback $callback Callback method, used to transform (highlighting) text. * @param array $params Array of additionall callback parameters passed through into it * (first non-optional parameter is an HTML fragment for highlighting) * @return string * @throws Zend_Search_Lucene_Exception */ public function highlightExtended($words, $callback, $params = array()) { if (!is_array($words)) { $words = array($words); } $wordsToHighlightList = array(); $analyzer = Zend_Search_Lucene_Analysis_Analyzer::getDefault(); foreach ($words as $wordString) { $wordsToHighlightList[] = $analyzer->tokenize($wordString); } $wordsToHighlight = call_user_func_array('array_merge', $wordsToHighlightList); if (count($wordsToHighlight) == 0) { return $this->_doc->saveHTML(); } $wordsToHighlightFlipped = array(); foreach ($wordsToHighlight as $id => $token) { $wordsToHighlightFlipped[$token->getTermText()] = $id; } if (!is_callable($callback)) { require_once 'Zend/Search/Lucene/Exception.php'; throw new Zend_Search_Lucene_Exception('$viewHelper parameter mast be a View Helper name, View Helper object or callback.'); } $xpath = new DOMXPath($this->_doc); $matchedNodes = $xpath->query("/html/body"); foreach ($matchedNodes as $matchedNode) { $this->_highlightNodeRecursive($matchedNode, $wordsToHighlightFlipped, $callback, $params); } } /** * Get HTML * * @return string */ public function getHTML() { return $this->_doc->saveHTML(); } /** * Get HTML body * * @return string */ public function getHtmlBody() { $xpath = new DOMXPath($this->_doc); $bodyNodes = $xpath->query('/html/body')->item(0)->childNodes; $outputFragments = array(); for ($count = 0; $count < $bodyNodes->length; $count++) { $outputFragments[] = $this->_doc->saveXML($bodyNodes->item($count)); } return implode($outputFragments); } }