#!/usr/bin/env bash
#############################################################################################
#THIS PROGRAM FETCHES NEWSLINKS FROM EUROPACIFIC CAPITAL. 
#[1]COVERTS THEM TO READABLE JAVASCRIPT CODE FOR INSERTING IN BLOGGER'S
#HTML/JAVASCRIPT WEB MODULE.
#[2]CREATES EXTERNAL HTML FILE FOR VIEWING.
##############################################################################################
#Change to working directory for crontab to work
cd /var/www/foreclosednation.com/public_html/rss 
#First we use curl to download the webpage.
curl -s http://www.europac.com/research_analysis/newswire > newswire.txt;
#
#Here we use egrep and sed on the file we just downloaded. We first create urls.txt file.
egrep -o 'url=([^'\'']*)' newswire.txt | sed 's|url=|<a href="|g' | sed 's|$|">|g' > urls.txt;
#Now let's grab the titles and create titles.txt file.
egrep -o "redirect\?url([^<]*)</a>" newswire.txt | sed -r 's|redirect\?url=[^>]*>||g' | sed -r 's|$|<p>|g' > titles.txt;
#remove http if doubled on same line (bug from europac)
sed -i -r 's/([^"])http.*/\1">/g' urls.txt;
#Merge urls and titles files and create t_fetch_news.txt
paste urls.txt titles.txt > t_fetch_news.txt;
#make an html version and keep older links
cat t_fetch_news.txt >> t_fetch_news.html;
#remove duplicate lines
awk '!seen[$0]++' t_fetch_news.html > tt_fetch_news.html; 
#add HTML TAGS
echo -e "<!DOCTYPE html><HEAD></HEAD><BODY>\n" > fetch_news.html;
cat tt_fetch_news.html >> fetch_news.html;
echo -e "\n</BODY></HTML>" >> fetch_news.html;
#remove smart quotes
sed -i -e 's/“/"/g; s/”/"/g; s/…/.../g; ' -i -e "s/’/'/g" -e "s/‘/'/g" fetch_news.html
#copy to main dir:
cp fetch_news.html ../index.html
#delete temp files
rm -rf t_fetch_news.html tt_fetch_news.html;
#remove lines and create fetch_news.txt
tr --delete '\n' < t_fetch_news.txt > fetch_news.txt;
#convert for javascript
sed -r 's|"|'\''|g' fetch_news.txt | sed -r 's|^|document.write("|g' | sed -r 's|$|");|g' > fetch_news.js