Thursday, 30 January 2020

JSSoup web scraping commodity price


extract data from first row of livestocks table

data extracted by JSSoup

//app.js

import React, { Component } from 'react';
import JSSoup from 'jssoup'
import axios from 'axios'
import { Image } from 'react-native';
import { TextInputMask } from 'react-native-masked-text'
import DrawerContent from './Drawer'
import CommodityListItem from './CommodityListItem'
import {
  Container, Header, Title, Content, Footer,
  FooterTab, Button, Left, Right, Body, Icon, Text,
  Accordion, Card, CardItem, Thumbnail, ListItem,
  CheckBox, DatePicker, DeckSwiper, View, Fab,
  Badge, Form, Item, Input, Label, Picker, Textarea,
  Switch, Radio, Spinner, Tab, Tabs, TabHeading,
  ScrollableTab, H1, H2, H3, Drawer,
} from 'native-base';

import { createIconSetFromIcoMoon } from '@expo/vector-icons';
import icon_json from './assets/selection.json'

const icon_ttf = require("./assets/icomoon.ttf");
const CustomIcon = createIconSetFromIcoMoon(icon_json, '', icon_ttf);

export default class App extends Component {
  constructor(props) {
    super(props);
    this.state = {
      category: 1,
      feeder_cattle: null
    };
  }

  componentDidMount() {

    axios({
      method: 'get',
      url: 'https://quotes.ino.com/exchanges/category.html?c=livestock',
    })
      .then(response => {
        //get source code from website
        const soup = new JSSoup(response.data)
        //find all rows with classname odd
        const tr = soup.findAll('tr', 'odd')
        //find all columns in the first row
        const td = tr[0].findAll('td')
        //extract column values and store them in a list
        let s = []
        td.forEach(element => {
          s.push(element.string.toString())
        });
        this.setState({ feeder_cattle: s })
        alert(this.state.feeder_cattle)
      })
      .catch(function (error) {
        alert(error);
      });
  }


  closeDrawer = () => {
    this.drawer._root.close()
  };

  openDrawer = () => {
    this.drawer._root.open()
  };

  changeCategory = (c) => {
    this.setState({ category: c })
  }

  render() {
    const { feeder_cattle } = this.state

    return (

      <Drawer ref={(ref) => { this.drawer = ref; }}
        content={
          <DrawerContent
            changeCategory={(c) => this.changeCategory(c)}
            closeDrawer={() => this.closeDrawer()}>
          </DrawerContent>
        }
        onClose={() => this.closeDrawer()} >

        <Container style={{ backgroundColor: '#1C2833' }}>
          <Header>
            <Left>
              <Button transparent onPress={() => this.openDrawer()}>
                <Icon name='menu' />
              </Button>
            </Left>
            <Body style={{ alignItems: 'center' }}>
              <Title>
                {
                  this.state.category === 1 ? 'Energy' :
                    this.state.category === 2 ? 'Metals' :
                      this.state.category === 3 ? 'Grains' :
                        this.state.category === 4 ? 'Softs' :
                          this.state.category === 5 ? 'Livestock' :
                            'drawer'
                }
              </Title>
            </Body>
            <Right></Right>
          </Header>

          <Content>

            <CommodityListItem
              iconName='feeder_cattle'
              title='Feeder Cattle'
              current={feeder_cattle ? feeder_cattle[5] : ''}
              dif={feeder_cattle ? feeder_cattle[6] : ''}
              percentDif={feeder_cattle ? feeder_cattle[7] : ''}
              open={feeder_cattle ? feeder_cattle[2] : ''}
              high={feeder_cattle ? feeder_cattle[3] : ''}
              low={feeder_cattle ? feeder_cattle[4] : ''}
              time={feeder_cattle ? feeder_cattle[8] + ' ' + feeder_cattle[1] : ''}
            ></CommodityListItem>
          </Content>
        </Container>
      </Drawer>
    );
  }
}

-------------------------------
CommodityListItem.js

import React, { Component } from 'react';
import {
    Container, Header, Title, Content, Footer,
    FooterTab, Button, Left, Right, Body, Icon, Text,
    Accordion, Card, CardItem, Thumbnail, ListItem,
    CheckBox, DatePicker, DeckSwiper, View, Fab,
    Badge, Form, Item, Input, Label, Picker, Textarea,
    Switch, Radio, Spinner, Tab, Tabs, TabHeading,
    ScrollableTab, H1, H2, H3, Drawer,
} from 'native-base';

import { createIconSetFromIcoMoon } from '@expo/vector-icons';
import icon_json from './assets/selection.json'

const icon_ttf = require("./assets/icomoon.ttf");
const CustomIcon = createIconSetFromIcoMoon(icon_json, '', icon_ttf);

export default class CommodityListItem extends Component {
    constructor(props) {
        super(props);
        this.state = {
        };
    }

    render() {
        return (
            <ListItem icon noBorder style={{ height: 100 }}>
                <Left><CustomIcon name={this.props.iconName} size={50} color="white"></CustomIcon></Left>
                <Body>
                    <Text style={{ color: 'gold' }}>{this.props.title}</Text>
                    <Text style={{ color: 'white' }}>{this.props.current}{' '}
                        {
                            this.props.dif.includes('-') ?
                                <CustomIcon name='down' size={15} color="red"></CustomIcon> :
                                <CustomIcon name='up' size={15} color="green"></CustomIcon>
                        }
                        {
                            this.props.dif.includes('-') ?
                                <Text style={{ color: 'red' }}>{this.props.dif} ({this.props.percentDif.replace('-', '')})</Text> :
                                <Text style={{ color: 'green' }}>{this.props.dif.replace('+', '')} ({this.props.percentDif.replace('+', '')})</Text>
                        }
                    </Text>
                    <Text note numberOfLines={2} style={{ color: 'white' }}>Open: {this.props.open}{'\n'}
                        High: {this.props.high} Low: {this.props.low}
                    </Text>
                </Body>
                <Right style={{ paddingTop: 50 }}>
                    <Text note style={{ color: 'white' }}>cent/pound{"\n"}{this.props.time}</Text>
                </Right>
            </ListItem>
        )
    }
}

------------------------------
//Drawer.js

import React, { Component } from 'react';
import {
    Container, Header, Title, Content, Footer,
    FooterTab, Button, Left, Right, Body, Icon, Text,
    Accordion, Card, CardItem, Thumbnail, ListItem,
    CheckBox, DatePicker, DeckSwiper, View, Fab,
    Badge, Form, Item, Input, Label, Picker, Textarea,
    Switch, Radio, Spinner, Tab, Tabs, TabHeading,
    ScrollableTab, H1, H2, H3, Drawer,
} from 'native-base';

import { createIconSetFromIcoMoon } from '@expo/vector-icons';
import icon_json from './assets/selection.json'

const icon_ttf = require("./assets/icomoon.ttf");
const CustomIcon = createIconSetFromIcoMoon(icon_json, '', icon_ttf);

export default class DrawerContent extends Component {
    constructor(props) {
        super(props);
        this.state = {
        };
    }

    render() {
        return (
            <Container style={{ backgroundColor: '#1C2833' }}>
                <Header>
                    <Body>
                        <Title>Commodity Price</Title>
                    </Body>
                    <Right>
                        <Button transparent onPress={() => this.props.closeDrawer()}>
                            <Icon name='arrow-back' />
                        </Button>
                    </Right>
                </Header>
                <Content>
                    <ListItem icon noBorder onPress={() => { this.props.changeCategory(1); this.props.closeDrawer() }}>
                        <Left>
                            <Icon type='FontAwesome5' name="fire" style={{ color: 'white' }} />
                        </Left>
                        <Body>
                            <Text style={{ color: 'white' }}>Energy</Text>
                        </Body>
                    </ListItem>
                    <ListItem icon noBorder onPress={() => { this.props.changeCategory(2); this.props.closeDrawer() }}>
                        <Left>
                            <CustomIcon name='gold' size={24} color="white"></CustomIcon>
                        </Left>
                        <Body>
                            <Text style={{ color: 'white' }}>Metals</Text>
                        </Body>
                    </ListItem>
                    <ListItem icon noBorder onPress={() => { this.props.changeCategory(3); this.props.closeDrawer() }}>
                        <Left>
                            <CustomIcon name='grain' size={24} color="white"></CustomIcon>
                        </Left>
                        <Body>
                            <Text style={{ color: 'white' }}>Grains</Text>
                        </Body>
                    </ListItem>
                    <ListItem icon noBorder onPress={() => { this.props.changeCategory(4); this.props.closeDrawer() }}>
                        <Left>
                            <CustomIcon name='coffee' size={24} color="white"></CustomIcon>
                        </Left>
                        <Body>
                            <Text style={{ color: 'white' }}>Softs</Text>
                        </Body>
                    </ListItem>
                    <ListItem icon noBorder onPress={() => { this.props.changeCategory(5); this.props.closeDrawer() }}>
                        <Left>
                            <CustomIcon name='live_cattle' size={24} color="white"></CustomIcon>
                        </Left>
                        <Body>
                            <Text style={{ color: 'white' }}>Livestock</Text>
                        </Body>
                    </ListItem>
                </Content>
            </Container>
        )
    }
}

-------------------------------------
//https://quotes.ino.com/exchanges/category.html?c=livestock souce code
...
<div class="table-responsive">
<table border="0" cellspacing="0" cellpadding="0" class="table table-condensed">
        <colgroup span="2" />
        <col />
        <col />
        <col />
        <col />
        <col />
        <col />
        <col />
<tr><th scope="col">Market</th><th scope="col">Contract</th><th class="rt" scope="col">Open</th><th class="rt" scope="col">High</th><th class="rt" scope="col">Low</th><th class="rt" scope="col">Last</th><th class="rt" scope="col">Change</th><th class="rt" scope="col">Pct</th><th class="rt" scope="col">Time</th></tr><tr><th colspan="5" scope="colgroup"><a href="contracts.html?r=CME_GF">FEEDER CATTLE (CME:GF)</a></th>
 <td class="rt util-links">
   <a href="https://club.ino.com/trend/?v=future&s=CME_GF&mktcode=INOFutures" target="signal" class="btn-entry-signal btn btn-sm btn-primary">Entry Signal</a></td>
 <td class="rt util-links" colspan="3">
<a href="contracts.html?r=CME_GF">View all months</a> | <a href="http://club.ino.com/members/download/data.html?s=CME_GF">Download data</a> | <a href="http://club.ino.com/members/search/?c=CME_GF&amp;mode=analysis">Analyze Chart</a></td></tr>
<tr class="odd"><td><a href="https://quotes.ino.com/charting/?s=CME_GF.F20">GF.F20</a></td><td><a href="https://quotes.ino.com/charting/?s=CME_GF.F20">Jan 2020</a></td><td class="rt">142.300</td><td class="rt">142.525</td><td class="rt">142.250</td><td class="rt">142.425</td><td class="rt up">+0.175</td><td class="rt up">+0.12%</td><td class="rt">17:28</td></tr><tr><td><a href="https://quotes.ino.com/charting/?s=CME_GF.H20">GF.H20</a></td><td><a href="https://quotes.ino.com/charting/?s=CME_GF.H20">Mar 2020</a></td><td class="rt">135.650</td><td class="rt">137.775</td><td class="rt">135.450</td><td class="rt">136.925</td><td class="rt up">+1.700</td><td class="rt up">+1.17%</td><td class="rt">12:00</td></tr><tr class="odd"><td><a href="https://quotes.ino.com/charting/?s=CME_GF.J20">GF.J20</a></td><td><a href="https://quotes.ino.com/charting/?s=CME_GF.J20">Apr 2020</a></td><td class="rt">137.200</td><td class="rt">139.475</td><td class="rt">137.075</td><td class="rt">138.625</td><td class="rt up">+1.725</td><td class="rt up">+1.16%</td><td class="rt">12:00</td></tr><tr><th colspan="9" align=center></th></tr>
<tr><th colspan="5" scope="colgroup"><a href="contracts.html?r=CME_HE">LEAN HOGS (CME:HE)</a></th>
 <td class="rt util-links">
   <a href="https://club.ino.com/trend/?v=future&s=CME_HE&mktcode=INOFutures" target="signal" class="btn-entry-signal btn btn-sm btn-primary">Entry Signal</a></td>
 <td class="rt util-links" colspan="3">
<a href="contracts.html?r=CME_HE">View all months</a> | <a href="http://club.ino.com/members/download/data.html?s=CME_HE">Download data</a> | <a href="http://club.ino.com/members/search/?c=CME_HE&amp;mode=analysis">Analyze Chart</a></td></tr>
<tr class="odd"><td><a href="https://quotes.ino.com/charting/?s=CME_HE.G20">HE.G20</a></td><td><a href="https://quotes.ino.com/charting/?s=CME_HE.G20">Feb 2020</a></td><td class="rt">60.50</td><td class="rt">60.50</td><td class="rt">57.35</td><td class="rt">58.85</td><td class="rt down">-2.45</td><td class="rt down">-3.72%</td><td class="rt">12:00</td></tr><tr>...

reference:
https://quotes.ino.com/

jssoup
https://www.npmjs.com/package/jssoup

JSSoup uses tautologistics/node-htmlparser as HTML dom parser, and creates a series of BeautifulSoup like API on top of it. JSSoup supports both node and react-native.

https://stackoverflow.com/questions/53770358/cannot-find-a-tag-with-jssoup-even-though-the-tag-exists-in-node-js
https://github.com/chishui/JSSoup/blob/94af47ba9209f4ed9c4b372c3f5cbbebfa679653/lib/jssoup.js#L153

https://stackoverflow.com/questions/41194866/how-to-set-state-of-response-from-axios-in-react
https://chuanshuoge2.blogspot.com/2020/01/nativebase-future-trade.html

No comments:

Post a Comment